From 3713cbfdacec24a098dc08527fbe75e6c07bb573 Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 15 May 2026 14:30:54 -0400 Subject: [PATCH 001/142] feat: Add DISJOIN operator for splitting intervals at breakpoints DISJOIN is a FROM-clause table function that cuts each target interval at every reference breakpoint strictly interior to it, so no resulting sub-interval partially overlaps a reference interval. The target row passes through unchanged and the sub-interval is appended as disjoin_chrom, disjoin_start and disjoin_end. A coverage filter drops sub-intervals overlapping no reference interval. When no reference is given it defaults to the target set, so selecting the distinct sub-intervals reproduces a Bioconductor-style disjoin. The generator emits a portable WITH-CTE subquery using only UNION, LEAD and EXISTS, so it transpiles unchanged to DuckDB, PostgreSQL and SQLite. Sub-interval boundaries are canonicalized to 0-based half-open regardless of the target or reference coordinate system. --- src/giql/dialect.py | 2 + src/giql/expressions.py | 30 ++++++ src/giql/generators/base.py | 195 +++++++++++++++++++++++++++++++++++- src/giql/transpile.py | 2 +- 4 files changed, 226 insertions(+), 3 deletions(-) diff --git a/src/giql/dialect.py b/src/giql/dialect.py index 6c70104..9098c51 100644 --- a/src/giql/dialect.py +++ b/src/giql/dialect.py @@ -13,6 +13,7 @@ from giql.expressions import Contains from giql.expressions import GIQLCluster +from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance from giql.expressions import GIQLMerge from giql.expressions import GIQLNearest @@ -57,6 +58,7 @@ class Parser(Parser): "MERGE": GIQLMerge.from_arg_list, "DISTANCE": GIQLDistance.from_arg_list, "NEAREST": GIQLNearest.from_arg_list, + "DISJOIN": GIQLDisjoin.from_arg_list, } def _parse_comparison(self): diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 857a223..c414dac 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -201,3 +201,33 @@ def from_arg_list(cls, args): if len(positional_args) >= 1: kwargs["this"] = positional_args[0] return cls(**kwargs) + + +class GIQLDisjoin(exp.Func): + """DISJOIN table function for splitting intervals at reference breakpoints. + + Generates SQL that cuts each target interval at every reference breakpoint + strictly interior to it, so each resulting sub-interval is fully contained + by every reference interval it overlaps. The target row passes through + intact and the sub-interval is appended as ``disjoin_chrom`` / + ``disjoin_start`` / ``disjoin_end``. When ``reference`` is omitted it + defaults to the target set. + + Examples: + DISJOIN(features) + DISJOIN(features, reference := other) + DISJOIN(features, reference => other) + DISJOIN(features, reference := (SELECT ...)) + """ + + arg_types = { + "this": True, # Required: target table name + "reference": False, # Optional: reference table/CTE name or subquery + } + + @classmethod + def from_arg_list(cls, args): + kwargs, positional_args = _split_named_and_positional(args) + if len(positional_args) >= 1: + kwargs["this"] = positional_args[0] + return cls(**kwargs) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 1b95b70..e53a5f5 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -6,6 +6,7 @@ from giql.constants import DEFAULT_START_COL from giql.constants import DEFAULT_STRAND_COL from giql.expressions import Contains +from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance from giql.expressions import GIQLNearest from giql.expressions import Intersects @@ -274,6 +275,93 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: return sql.strip() + def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: + """Generate SQL for the DISJOIN table function. + + DISJOIN splits each target interval at every reference breakpoint + strictly interior to it. The full target row passes through unchanged; + the sub-interval is appended as ``disjoin_chrom`` / ``disjoin_start`` / + ``disjoin_end`` in the target table's coordinate system. A coverage + filter drops sub-intervals overlapping no reference interval. When no + ``reference`` is given it defaults to the target set. + + :param expression: + GIQLDisjoin expression node + :return: + SQL string (a parenthesized WITH-CTE subquery) for the DISJOIN table + """ + # Resolve the target table and its genomic columns. + target_name, (target_chrom, target_start, target_end) = ( + self._resolve_target_table(expression) + ) + target_table = self.tables.get(target_name) if self.tables else None + + # Resolve the reference relation (defaults to the target set). + ref_from, ref_chrom, ref_start, ref_end, ref_table = ( + self._resolve_disjoin_reference( + expression, + target_name, + (target_chrom, target_start, target_end), + target_table, + ) + ) + + # Canonical target endpoints, qualified by the __giql_dj_tgt alias. + t_chrom = f't."{target_chrom}"' + t_start = self._canonical_start(f't."{target_start}"', target_table) + t_end = self._canonical_end(f't."{target_end}"', target_table) + + # Canonical reference endpoints: unqualified for the breakpoint CTE, + # qualified by 'r' for the coverage EXISTS filter. + bp_start = self._canonical_start(f'"{ref_start}"', ref_table) + bp_end = self._canonical_end(f'"{ref_end}"', ref_table) + r_start = self._canonical_start(f'r."{ref_start}"', ref_table) + r_end = self._canonical_end(f'r."{ref_end}"', ref_table) + + # disjoin_start / disjoin_end are emitted in the target table's + # coordinate system so an output row carries one convention; the cut + # math above stays canonical internally. + out_start = self._decanonical_start("s.seg_start", target_table) + out_end = self._decanonical_end("s.seg_end", target_table) + + # The `seg_end > seg_start` guard in the final WHERE is belt-and- + # suspenders: UNION already dedupes cut positions, so LEAD cannot + # produce a zero-length segment unless that UNION becomes UNION ALL. + return ( + "(WITH __giql_dj_ref AS (" + f"SELECT * FROM {ref_from}" + "), __giql_dj_tgt AS (" + f"SELECT * FROM {target_name}" + "), __giql_dj_bp AS (" + f'SELECT "{ref_chrom}" AS chrom, {bp_start} AS pos FROM __giql_dj_ref' + " UNION " + f'SELECT "{ref_chrom}" AS chrom, {bp_end} AS pos FROM __giql_dj_ref' + "), __giql_dj_cuts AS (" + f'SELECT t."{target_chrom}" AS kc, t."{target_start}" AS ks,' + f' t."{target_end}" AS ke, {t_start} AS pos FROM __giql_dj_tgt AS t' + " UNION " + f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}",' + f" {t_end} FROM __giql_dj_tgt AS t" + " UNION " + f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}",' + " bp.pos FROM __giql_dj_tgt AS t JOIN __giql_dj_bp AS bp" + f" ON bp.chrom = {t_chrom} AND bp.pos > {t_start}" + f" AND bp.pos < {t_end}" + "), __giql_dj_segs AS (" + "SELECT kc, ks, ke, pos AS seg_start," + " LEAD(pos) OVER (PARTITION BY kc, ks, ke ORDER BY pos) AS seg_end" + " FROM __giql_dj_cuts" + ") SELECT t.*, s.kc AS disjoin_chrom," + f" {out_start} AS disjoin_start, {out_end} AS disjoin_end" + " FROM __giql_dj_tgt AS t JOIN __giql_dj_segs AS s" + f' ON t."{target_chrom}" = s.kc AND t."{target_start}" = s.ks' + f' AND t."{target_end}" = s.ke' + " WHERE s.seg_end IS NOT NULL AND s.seg_end > s.seg_start" + " AND EXISTS (SELECT 1 FROM __giql_dj_ref AS r WHERE" + f' r."{ref_chrom}" = s.kc AND {r_start} <= s.seg_start' + f" AND {r_end} > s.seg_start))" + ) + def giqldistance_sql(self, expression: GIQLDistance) -> str: """Generate SQL CASE expression for DISTANCE function. @@ -767,12 +855,12 @@ def _resolve_nearest_reference( return self._canonical_endpoints(chrom, start, end, table) def _resolve_target_table( - self, expression: GIQLNearest + self, expression: GIQLNearest | GIQLDisjoin ) -> tuple[str, tuple[str, str, str]]: """Resolve the target table name and its genomic column references. :param expression: - GIQLNearest expression node + GIQLNearest or GIQLDisjoin expression node :return: Tuple of (table_name, (chromosome_col, start_col, end_col)) :raises ValueError: @@ -800,6 +888,75 @@ def _resolve_target_table( # Get physical column names from table config return table_name, (table.chrom_col, table.start_col, table.end_col) + def _resolve_disjoin_reference( + self, + expression: GIQLDisjoin, + target_name: str, + target_cols: tuple[str, str, str], + target_table: Table | None, + ) -> tuple[str, str, str, str, Table | None]: + """Resolve the reference relation for a DISJOIN query. + + The reference contributes the breakpoints at which target intervals are + cut. When ``reference`` is omitted it defaults to the target set. + + A bare reference name is resolved against the registered tables + first: a match contributes that table's column names and coordinate + system. A name with no registered table is treated as a CTE and + assumed canonical. A CTE that shares a registered table's name will + therefore inherit that table's configuration -- avoid such collisions. + + :param expression: + GIQLDisjoin expression node + :param target_name: + Resolved target table name (the default reference) + :param target_cols: + ``(chrom, start, end)`` column names of the target table + :param target_table: + Table config of the target (the default reference config) + :return: + Tuple of ``(from_clause, chrom_col, start_col, end_col, table)`` + where ``from_clause`` is the text following ``FROM`` inside the + ``__giql_dj_ref`` CTE. A subquery or unregistered (CTE) reference + is assumed to expose canonical 0-based half-open ``chrom`` / + ``start`` / ``end`` columns, so ``table`` is ``None`` for those. + """ + reference = expression.args.get("reference") + tc, ts, te = target_cols + + # No reference: default to the target set (single-mode). + if reference is None: + return target_name, tc, ts, te, target_table + + # Subquery reference: inline it as an aliased derived table. + if isinstance(reference, exp.Subquery): + from_clause = f"{self.sql(reference)} AS __giql_dj_rs" + return ( + from_clause, + DEFAULT_CHROM_COL, + DEFAULT_START_COL, + DEFAULT_END_COL, + None, + ) + + # Bare table or CTE name. + if isinstance(reference, (exp.Table, exp.Column)): + ref_name = reference.name + else: + ref_name = self.sql(reference).strip('"') + + ref_table = self.tables.get(ref_name) if self.tables else None + if ref_table: + return ( + ref_name, + ref_table.chrom_col, + ref_table.start_col, + ref_table.end_col, + ref_table, + ) + # Unregistered name (e.g. a CTE): assume canonical default columns. + return ref_name, DEFAULT_CHROM_COL, DEFAULT_START_COL, DEFAULT_END_COL, None + def _get_column_refs( self, column_ref: str, @@ -892,6 +1049,40 @@ def _canonical_end(raw_end: str, table: Table | None) -> str: return f"({raw_end} - 1)" return raw_end # 0based/half_open and 1based/closed: identity + @staticmethod + def _decanonical_start(canonical_start: str, table: Table | None) -> str: + """Convert a canonical 0-based half-open start to the table's encoding. + + Inverse of ``_canonical_start``. Conversion by ``coordinate_system``: + + - 0based: ``start`` (identity) + - 1based: ``start + 1`` + """ + if table is None or table.coordinate_system == "0based": + return canonical_start + return f"({canonical_start} + 1)" + + @staticmethod + def _decanonical_end(canonical_end: str, table: Table | None) -> str: + """Convert a canonical 0-based half-open end to the table's encoding. + + Inverse of ``_canonical_end``. Conversion by ``(coordinate_system, + interval_type)``: + + - 0based / half_open: ``end`` (identity) + - 0based / closed: ``end - 1`` + - 1based / half_open: ``end + 1`` + - 1based / closed: ``end`` (identity) + """ + if table is None: + return canonical_end + key = (table.coordinate_system, table.interval_type) + if key == ("0based", "closed"): + return f"({canonical_end} - 1)" + if key == ("1based", "half_open"): + return f"({canonical_end} + 1)" + return canonical_end # 0based/half_open and 1based/closed: identity + @staticmethod def _canonical_endpoints( chrom: str, diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 7c70746..94eea2a 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -58,7 +58,7 @@ def transpile( ---------- giql : str The GIQL query string containing genomic extensions like - INTERSECTS, CONTAINS, WITHIN, CLUSTER, MERGE, or NEAREST. + INTERSECTS, CONTAINS, WITHIN, CLUSTER, MERGE, NEAREST, or DISJOIN. tables : list[str | Table] | None Table configurations. Strings use default column mappings (chrom, start, end, strand). Table objects provide custom From e7b6b12c68032fb529642481200b16ab7c619d2b Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 15 May 2026 14:31:01 -0400 Subject: [PATCH 002/142] feat: Register DISJOIN in the MCP operator catalog Add DISJOIN to the MCP server operator metadata and documentation path map so it is discoverable through list_operators, explain_operator and search_docs. The list_operators count assertion is updated to match the new operator total. --- src/giql/mcp/server.py | 27 +++++++++++++++++++++++++-- tests/test_mcp_server.py | 4 ++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/giql/mcp/server.py b/src/giql/mcp/server.py index 8f74ae6..1de3565 100644 --- a/src/giql/mcp/server.py +++ b/src/giql/mcp/server.py @@ -123,6 +123,21 @@ "example": "SELECT MERGE(interval), COUNT(*) FROM features", "doc_file": "dialect/aggregation-operators.rst", }, + "DISJOIN": { + "category": "set-operation", + "description": "Split genomic intervals into sub-intervals at reference breakpoints", + "syntax": "SELECT * FROM DISJOIN(target, reference := refs)", + "parameters": [ + {"name": "target", "description": "Table of intervals to split"}, + { + "name": "reference", + "description": "Table, CTE, or subquery supplying breakpoints (defaults to target)", + }, + ], + "returns": "Each target row with the sub-interval appended (disjoin_chrom, disjoin_start, disjoin_end)", + "example": "SELECT * FROM DISJOIN(features, reference := mask)", + "doc_file": "dialect/aggregation-operators.rst", + }, "ANY": { "category": "quantifier", "description": "Match if condition holds for any of the specified ranges", @@ -216,6 +231,7 @@ def find_docs_root() -> Path | None: "recipes/intersect": "recipes/intersect.rst", "recipes/distance": "recipes/distance.rst", "recipes/clustering": "recipes/clustering.rst", + "recipes/disjoin": "recipes/disjoin.rst", "recipes/bedtools-migration": "recipes/bedtools-migration.rst", "recipes/advanced": "recipes/advanced.rst", } @@ -292,7 +308,8 @@ def get_documentation(path: str) -> str: transpilation/execution, transpilation/performance, transpilation/schema-mapping - recipes/index, recipes/intersect, recipes/distance, - recipes/clustering, recipes/bedtools-migration, recipes/advanced + recipes/clustering, recipes/disjoin, recipes/bedtools-migration, + recipes/advanced """ if path not in DOC_PATHS: available = ", ".join(sorted(DOC_PATHS.keys())) @@ -318,6 +335,7 @@ def list_operators() -> list[dict[str, str]]: - Spatial: INTERSECTS, CONTAINS, WITHIN - Distance: DISTANCE, NEAREST - Aggregation: CLUSTER, MERGE + - Set operations: DISJOIN - Quantifiers: ANY, ALL """ result = [] @@ -340,7 +358,7 @@ def explain_operator(name: str) -> dict[str, Any]: Args: name: Operator name (case-insensitive). One of: INTERSECTS, CONTAINS, WITHIN, DISTANCE, NEAREST, - CLUSTER, MERGE, ANY, ALL + CLUSTER, MERGE, DISJOIN, ANY, ALL """ name_upper = name.upper().strip() @@ -430,6 +448,11 @@ def get_syntax_reference() -> str: MERGE - Combine overlapping intervals SELECT MERGE(interval) FROM table +Set Operations +-------------- +DISJOIN - Split intervals at reference breakpoints + SELECT * FROM DISJOIN(target, reference := refs) + Set Quantifiers --------------- ANY - Match any of multiple ranges diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 1a7791d..ee429b8 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -184,10 +184,10 @@ def test_returns_all_operators(self): """ GIVEN the GIQL MCP server WHEN list_operators is called - THEN it returns all 9 operators + THEN it returns all 10 operators """ result = list_operators() - assert len(result) == 9 + assert len(result) == 10 def test_each_has_required_fields(self): """ From 305df32416db71c3f206eee66f0cdc5f83c8e02e Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 15 May 2026 14:31:09 -0400 Subject: [PATCH 003/142] test: Cover DISJOIN parsing, transpilation, and execution Add parsing tests for the GIQLDisjoin AST node, transpilation tests for the generated SQL shape, end-to-end execution tests on DuckDB, and a coordinate-space matrix asserting convention-invariant results across all four coordinate-system and interval-type encodings. --- pyproject.toml | 3 + .../test_disjoin_coordinate_space.py | 114 ++++++++++ tests/test_disjoin_parsing.py | 142 +++++++++++++ tests/test_disjoin_transpilation.py | 127 ++++++++++++ tests/test_disjoin_udf.py | 196 ++++++++++++++++++ 5 files changed, 582 insertions(+) create mode 100644 tests/integration/coordinate_space/test_disjoin_coordinate_space.py create mode 100644 tests/test_disjoin_parsing.py create mode 100644 tests/test_disjoin_transpilation.py create mode 100644 tests/test_disjoin_udf.py diff --git a/pyproject.toml b/pyproject.toml index 647358b..8203b1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,9 @@ path = "build-hooks/metadata.py" [tool.pytest.ini_options] addopts = "--cov --cov-config=.coveragerc" +markers = [ + "integration: end-to-end tests that execute generated SQL against a real database", +] [tool.ruff] line-length = 89 diff --git a/tests/integration/coordinate_space/test_disjoin_coordinate_space.py b/tests/integration/coordinate_space/test_disjoin_coordinate_space.py new file mode 100644 index 0000000..e15e58a --- /dev/null +++ b/tests/integration/coordinate_space/test_disjoin_coordinate_space.py @@ -0,0 +1,114 @@ +"""Integration tests asserting DISJOIN is convention-invariant. + +For a fixed set of canonical 0-based half-open intervals, encoding the same +logical intervals under any of the four ``(coordinate_system, interval_type)`` +combinations -- on the target, the reference, or both -- must produce the same +logical partition. DISJOIN emits ``disjoin_start`` / ``disjoin_end`` in the +target table's coordinate system, so the expected rows are re-encoded under +that convention. +""" + +import pytest + +from .encodings import CONVENTIONS +from .encodings import encode +from .encodings import make_table + +pytestmark = pytest.mark.integration + + +def _row(chrom: str, start: int, end: int, name: str) -> tuple: + """Build a 6-tuple suitable for ``load_intervals`` with strand ``+``.""" + return (chrom, start, end, name, 100, "+") + + +class TestDisjoinCoordinateSpace: + """Convention-invariance of DISJOIN end-to-end via DuckDB.""" + + @pytest.mark.parametrize( + ("coordinate_system", "interval_type"), + CONVENTIONS, + ids=lambda v: str(v), + ) + def test_disjoin_should_yield_the_partition_in_the_target_encoding( + self, giql_query, coordinate_system, interval_type + ): + """Test DISJOIN self-mode yields the partition in the target's encoding. + + Given: + Two overlapping intervals (canonical [0, 20) and [10, 30)) + re-encoded under one convention. + When: + DISJOIN(features) runs in self-mode and DISTINCT sub-intervals + are selected. + Then: + The sub-intervals should be the partition {[0, 10), [10, 20), + [20, 30)} re-encoded under the target table's convention. + """ + # Arrange + s_a, e_a = encode(0, 20, coordinate_system, interval_type) + s_b, e_b = encode(10, 30, coordinate_system, interval_type) + tables = [make_table("features", coordinate_system, interval_type)] + + # Act + result = giql_query( + "SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end " + "FROM DISJOIN(features) ORDER BY disjoin_start", + tables=tables, + features=[_row("chr1", s_a, e_a, "a"), _row("chr1", s_b, e_b, "b")], + ) + + # Assert + expected = [ + ("chr1", *encode(0, 10, coordinate_system, interval_type)), + ("chr1", *encode(10, 20, coordinate_system, interval_type)), + ("chr1", *encode(20, 30, coordinate_system, interval_type)), + ] + assert result == expected + + @pytest.mark.parametrize( + ("coordinate_system", "interval_type"), + CONVENTIONS, + ids=lambda v: str(v), + ) + def test_disjoin_should_yield_target_encoded_subintervals_when_reference_differs( + self, giql_query, coordinate_system, interval_type + ): + """Test DISJOIN emits target-encoded sub-intervals across mixed encodings. + + Given: + A target interval (canonical [0, 30)) under one convention and a + reference set (canonical [0, 10) and [10, 30)) under a fixed, + different convention. + When: + DISJOIN(features, reference := refs) runs. + Then: + The sub-intervals {[0, 10), [10, 30)} should be re-encoded under + the target table's convention regardless of either side's storage + encoding. + """ + # Arrange + ref_system, ref_type = "1based", "closed" + s_t, e_t = encode(0, 30, coordinate_system, interval_type) + r1s, r1e = encode(0, 10, ref_system, ref_type) + r2s, r2e = encode(10, 30, ref_system, ref_type) + tables = [ + make_table("features", coordinate_system, interval_type), + make_table("refs", ref_system, ref_type), + ] + + # Act + result = giql_query( + "SELECT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", + tables=tables, + features=[_row("chr1", s_t, e_t, "t")], + refs=[_row("chr1", r1s, r1e, "r1"), _row("chr1", r2s, r2e, "r2")], + ) + + # Assert + expected = [ + encode(0, 10, coordinate_system, interval_type), + encode(10, 30, coordinate_system, interval_type), + ] + assert result == expected diff --git a/tests/test_disjoin_parsing.py b/tests/test_disjoin_parsing.py new file mode 100644 index 0000000..5480445 --- /dev/null +++ b/tests/test_disjoin_parsing.py @@ -0,0 +1,142 @@ +"""Parser tests for the DISJOIN operator syntax. + +Tests verify that the GIQL parser recognizes DISJOIN table-function calls +and maps positional and named arguments onto the GIQLDisjoin AST node. +""" + +from sqlglot import exp +from sqlglot import parse_one + +from giql.dialect import GIQLDialect +from giql.expressions import GIQLDisjoin + + +def _disjoin_node(sql: str) -> GIQLDisjoin: + """Parse a GIQL query and return its first GIQLDisjoin node.""" + ast = parse_one(sql, dialect=GIQLDialect) + node = ast.find(GIQLDisjoin) + assert node is not None, f"Expected a GIQLDisjoin node in: {sql}" + return node + + +class TestDisjoinParsing: + """Tests for parsing DISJOIN function syntax.""" + + def test_from_arg_list_should_expose_documented_arg_types(self): + """Test that GIQLDisjoin declares the documented argument schema. + + Given: + The GIQLDisjoin expression class. + When: + Inspecting its arg_types mapping. + Then: + It should require `this` and make `reference` optional. + """ + # Arrange, act, & assert + assert GIQLDisjoin.arg_types == {"this": True, "reference": False} + + def test_from_arg_list_should_map_positional_arg_to_target(self): + """Test that a positional argument binds to the target set. + + Given: + A GIQL query with DISJOIN(features). + When: + Parsing the query. + Then: + It should create a GIQLDisjoin node with the target set as `this`. + """ + # Arrange & act + node = _disjoin_node("SELECT * FROM DISJOIN(features)") + + # Assert + assert isinstance(node, GIQLDisjoin) + assert node.this is not None + assert node.this.name == "features" + + def test_from_arg_list_should_leave_reference_unset_when_omitted(self): + """Test that an omitted reference argument stays unset. + + Given: + A GIQL query with DISJOIN(features) and no reference argument. + When: + Parsing the query. + Then: + It should leave the GIQLDisjoin node's reference argument unset. + """ + # Arrange & act + node = _disjoin_node("SELECT * FROM DISJOIN(features)") + + # Assert + assert node.args.get("reference") is None + + def test_from_arg_list_should_ignore_extra_positional_argument(self): + """Test that a bare second positional argument is not bound. + + Given: + A GIQL query with DISJOIN(features, refs) using a bare second + positional argument. + When: + Parsing the query. + Then: + It should bind `this` to features and leave `reference` unset. + """ + # Arrange & act + node = _disjoin_node("SELECT * FROM DISJOIN(features, refs)") + + # Assert + assert node.this.name == "features" + assert node.args.get("reference") is None + + def test_from_arg_list_should_map_reference_when_named_with_walrus(self): + """Test that a walrus-named reference argument is carried. + + Given: + A GIQL query with DISJOIN(features, reference := refs). + When: + Parsing the query. + Then: + It should carry the reference argument on the GIQLDisjoin node. + """ + # Arrange & act + node = _disjoin_node("SELECT * FROM DISJOIN(features, reference := refs)") + + # Assert + reference = node.args.get("reference") + assert reference is not None + assert reference.name == "refs" + + def test_from_arg_list_should_map_reference_when_named_with_arrow(self): + """Test that an arrow-named reference argument is carried. + + Given: + A GIQL query with DISJOIN(features, reference => refs). + When: + Parsing the query. + Then: + It should carry the reference argument on the GIQLDisjoin node. + """ + # Arrange & act + node = _disjoin_node("SELECT * FROM DISJOIN(features, reference => refs)") + + # Assert + assert node.args.get("reference") is not None + + def test_from_arg_list_should_carry_subquery_reference(self): + """Test that a subquery reference is carried as a nested SELECT. + + Given: + A GIQL query whose DISJOIN reference is a subquery. + When: + Parsing the query. + Then: + It should carry a reference argument containing a nested SELECT. + """ + # Arrange & act + node = _disjoin_node( + "SELECT * FROM DISJOIN(features, reference := (SELECT * FROM refs))" + ) + + # Assert + reference = node.args.get("reference") + assert reference is not None + assert reference.find(exp.Select) is not None diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py new file mode 100644 index 0000000..6844ee8 --- /dev/null +++ b/tests/test_disjoin_transpilation.py @@ -0,0 +1,127 @@ +"""Transpilation tests for DISJOIN SQL generation. + +Tests verify that DISJOIN() is transpiled to the expected WITH-CTE subquery +shape and that target/reference resolution and coordinate-system +canonicalization are reflected in the generated SQL. +""" + +from giql import transpile +from giql.table import Table + + +class TestDisjoinTranspilation: + """Tests for DISJOIN transpilation to SQL.""" + + def test_giqldisjoin_sql_should_emit_with_cte_subquery(self): + """ + GIVEN a GIQL query selecting from DISJOIN(features) + WHEN transpiling to SQL + THEN the output should be a WITH-CTE subquery using UNION, LEAD and EXISTS + """ + sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]).upper() + + assert "WITH __GIQL_DJ_REF" in sql + assert "UNION" in sql + assert "LEAD(" in sql + assert "EXISTS (" in sql + + def test_giqldisjoin_sql_should_emit_disjoin_columns(self): + """ + GIVEN a GIQL query selecting from DISJOIN(features) + WHEN transpiling to SQL + THEN the sub-interval should be appended as disjoin_chrom/start/end + """ + sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) + + assert "AS disjoin_chrom" in sql + assert "AS disjoin_start" in sql + assert "AS disjoin_end" in sql + + def test_giqldisjoin_sql_should_default_reference_to_target_when_omitted(self): + """ + GIVEN a DISJOIN call with no reference argument + WHEN transpiling to SQL + THEN the reference CTE should select from the target table + """ + sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) + + assert "__giql_dj_ref AS (SELECT * FROM features)" in sql + + def test_giqldisjoin_sql_should_use_explicit_reference_relation(self): + """ + GIVEN a DISJOIN call with an explicit reference table + WHEN transpiling to SQL + THEN the reference CTE should select from that reference table + """ + sql = transpile( + "SELECT * FROM DISJOIN(features, reference := refs)", + tables=["features", "refs"], + ) + + assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql + + def test_giqldisjoin_sql_should_honor_custom_column_names(self): + """ + GIVEN a target table with custom genomic column names + WHEN transpiling a DISJOIN query + THEN the generated SQL should reference the custom column names + """ + sql = transpile( + "SELECT * FROM DISJOIN(feats)", + tables=[Table("feats", chrom_col="seqid", start_col="lo", end_col="hi")], + ) + + assert 't."seqid"' in sql + assert 't."lo"' in sql + assert 't."hi"' in sql + + def test_giqldisjoin_sql_should_canonicalize_one_based_closed_target(self): + """ + GIVEN a target table stored as 1-based closed intervals + WHEN transpiling a DISJOIN query + THEN the start should be shifted to canonical 0-based coordinates + """ + sql = transpile( + "SELECT * FROM DISJOIN(features)", + tables=[ + Table( + "features", + coordinate_system="1based", + interval_type="closed", + ) + ], + ) + + assert '(t."start" - 1)' in sql + + def test_giqldisjoin_sql_should_emit_disjoin_start_in_target_encoding(self): + """ + GIVEN a target table stored as 1-based closed intervals + WHEN transpiling a DISJOIN query + THEN disjoin_start should be shifted back to the target's encoding + """ + sql = transpile( + "SELECT * FROM DISJOIN(features)", + tables=[ + Table( + "features", + coordinate_system="1based", + interval_type="closed", + ) + ], + ) + + assert "(s.seg_start + 1) AS disjoin_start" in sql + + def test_giqldisjoin_sql_should_inline_subquery_reference(self): + """ + GIVEN a DISJOIN call whose reference is a subquery + WHEN transpiling to SQL + THEN the subquery should be inlined as an aliased derived table + """ + sql = transpile( + "SELECT * FROM DISJOIN(features, reference := (SELECT * FROM refs))", + tables=["features", "refs"], + ) + + assert "AS __giql_dj_rs" in sql diff --git a/tests/test_disjoin_udf.py b/tests/test_disjoin_udf.py new file mode 100644 index 0000000..48b5d7a --- /dev/null +++ b/tests/test_disjoin_udf.py @@ -0,0 +1,196 @@ +"""End-to-end execution tests for the DISJOIN operator on DuckDB. + +Tests transpile DISJOIN queries and execute them against in-memory DuckDB +to verify split correctness, the coverage filter, parent passthrough, and +degenerate-input handling. +""" + +import duckdb + +from giql import transpile + + +def _run(query: str, tables: list[str], **table_data): + """Transpile a DISJOIN query and execute it against in-memory DuckDB.""" + conn = duckdb.connect(":memory:") + for name, rows in table_data.items(): + conn.execute( + f'CREATE TABLE {name}(chrom VARCHAR, "start" INTEGER, ' + f'"end" INTEGER, name VARCHAR)' + ) + conn.executemany(f"INSERT INTO {name} VALUES (?, ?, ?, ?)", rows) + result = conn.execute(transpile(query, tables=tables)).fetchall() + conn.close() + return result + + +class TestDisjoinExecution: + """End-to-end DISJOIN correctness on DuckDB.""" + + def test_disjoin_should_split_intervals_at_breakpoints(self): + """ + GIVEN two overlapping target intervals A=[0,20) and B=[10,30) + WHEN DISJOIN runs in self-mode + THEN each interval should be split at the other's interior breakpoint + """ + result = _run( + "SELECT name, disjoin_start, disjoin_end FROM DISJOIN(features) " + "ORDER BY name, disjoin_start", + tables=["features"], + features=[("chr1", 0, 20, "A"), ("chr1", 10, 30, "B")], + ) + + assert result == [ + ("A", 0, 10), + ("A", 10, 20), + ("B", 10, 20), + ("B", 20, 30), + ] + + def test_disjoin_should_yield_bioconductor_partition_when_distinct(self): + """ + GIVEN overlapping target intervals in self-mode + WHEN selecting DISTINCT sub-intervals from DISJOIN + THEN the result should be the globally non-overlapping partition + """ + result = _run( + "SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end " + "FROM DISJOIN(features) ORDER BY disjoin_start", + tables=["features"], + features=[("chr1", 0, 20, "A"), ("chr1", 10, 30, "B")], + ) + + assert result == [("chr1", 0, 10), ("chr1", 10, 20), ("chr1", 20, 30)] + + def test_disjoin_should_split_against_explicit_reference(self): + """ + GIVEN a target interval and a separate reference set + WHEN DISJOIN runs with an explicit reference + THEN the target should be cut at the reference breakpoints + """ + result = _run( + "SELECT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", + tables=["features", "refs"], + features=[("chr1", 0, 30, "T")], + refs=[("chr1", 0, 10, "a"), ("chr1", 10, 30, "b")], + ) + + assert result == [(0, 10), (10, 30)] + + def test_disjoin_should_drop_pieces_overlapping_no_reference(self): + """ + GIVEN a target spanning a gap in the reference coverage + WHEN DISJOIN runs with an explicit reference + THEN sub-intervals overlapping no reference interval should be dropped + """ + result = _run( + "SELECT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", + tables=["features", "refs"], + features=[("chr1", 0, 30, "T")], + refs=[("chr1", 0, 10, "a"), ("chr1", 20, 30, "b")], + ) + + assert result == [(0, 10), (20, 30)] + + def test_disjoin_should_yield_no_rows_when_target_is_a_point(self): + """ + GIVEN a zero-length target interval + WHEN DISJOIN runs + THEN it should produce no output rows + """ + result = _run( + "SELECT * FROM DISJOIN(features)", + tables=["features"], + features=[("chr1", 5, 5, "P")], + ) + + assert result == [] + + def test_disjoin_should_split_duplicate_targets_independently(self): + """ + GIVEN two target rows with identical geometry + WHEN DISJOIN runs with a reference that cuts them + THEN each duplicate row should be split independently + """ + result = _run( + "SELECT name, disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := refs) ORDER BY name, disjoin_start", + tables=["features", "refs"], + features=[("chr1", 0, 10, "X"), ("chr1", 0, 10, "Y")], + refs=[("chr1", 0, 5, "a"), ("chr1", 5, 10, "b")], + ) + + assert result == [("X", 0, 5), ("X", 5, 10), ("Y", 0, 5), ("Y", 5, 10)] + + def test_disjoin_should_not_cross_chromosome_boundaries(self): + """ + GIVEN target intervals on different chromosomes + WHEN DISJOIN runs in self-mode + THEN sub-intervals should never span a chromosome boundary + """ + result = _run( + "SELECT disjoin_chrom, disjoin_start, disjoin_end FROM DISJOIN(features) " + "ORDER BY disjoin_chrom, disjoin_start", + tables=["features"], + features=[("chr1", 0, 20, "A"), ("chr2", 5, 25, "B")], + ) + + assert result == [("chr1", 0, 20), ("chr2", 5, 25)] + + def test_disjoin_should_pass_through_non_interval_columns(self): + """ + GIVEN a target table carrying a non-interval column + WHEN DISJOIN runs + THEN every output row should carry the intact parent row alongside + its sub-interval + """ + result = _run( + 'SELECT name, chrom, "start", "end", disjoin_start, disjoin_end ' + "FROM DISJOIN(features) ORDER BY disjoin_start, name", + tables=["features"], + features=[("chr1", 0, 20, "A"), ("chr1", 10, 30, "B")], + ) + + assert result == [ + ("A", "chr1", 0, 20, 0, 10), + ("A", "chr1", 0, 20, 10, 20), + ("B", "chr1", 10, 30, 10, 20), + ("B", "chr1", 10, 30, 20, 30), + ] + + def test_disjoin_should_accept_a_cte_as_reference(self): + """ + GIVEN a reference supplied as a CTE defined in the outer query + WHEN DISJOIN runs against that CTE + THEN the target should be split at the CTE's breakpoints + """ + result = _run( + "WITH bins AS (" + " SELECT 'chr1' AS chrom, 0 AS \"start\", 10 AS \"end\" " + " UNION ALL SELECT 'chr1', 10, 20" + ") " + "SELECT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := bins) ORDER BY disjoin_start", + tables=["features"], + features=[("chr1", 0, 20, "T")], + ) + + assert result == [(0, 10), (10, 20)] + + def test_disjoin_should_not_cut_at_breakpoint_on_target_boundary(self): + """ + GIVEN a reference breakpoint coinciding with a target boundary + WHEN DISJOIN runs + THEN the target should not be split at that boundary + """ + result = _run( + "SELECT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := refs)", + tables=["features", "refs"], + features=[("chr1", 10, 20, "T")], + refs=[("chr1", 10, 20, "r")], + ) + + assert result == [(10, 20)] From 959237c1da9dfeaee158e1820cae2f6dc019ccfc Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 15 May 2026 14:31:18 -0400 Subject: [PATCH 004/142] docs: Document the DISJOIN operator Add a DISJOIN section to the aggregation operators reference and a new disjoin recipe covering set partitioning and reference-grid splitting, wired into the recipes index and toctree. --- docs/dialect/aggregation-operators.rst | 129 +++++++++++++++++++++++++ docs/index.rst | 1 + docs/recipes/disjoin.rst | 81 ++++++++++++++++ docs/recipes/index.rst | 4 + 4 files changed, 215 insertions(+) create mode 100644 docs/recipes/disjoin.rst diff --git a/docs/dialect/aggregation-operators.rst b/docs/dialect/aggregation-operators.rst index 9887b87..2d488bf 100644 --- a/docs/dialect/aggregation-operators.rst +++ b/docs/dialect/aggregation-operators.rst @@ -329,3 +329,132 @@ Related Operators - :ref:`CLUSTER ` - Assign cluster IDs without merging - :ref:`INTERSECTS ` - Test for overlap between specific pairs + +---- + +.. _disjoin-operator: + +DISJOIN +------- + +Split genomic intervals at reference breakpoints into sub-intervals that +never partially overlap a reference interval. + +Description +~~~~~~~~~~~ + +The ``DISJOIN`` operator is a table function: it takes a set of *target* +intervals and a *reference* set of intervals, and cuts each target interval +at every reference breakpoint (a reference ``start`` or ``end`` position) +that falls strictly inside it. Every resulting sub-interval is fully +contained by each reference interval it overlaps -- it can never *partially* +overlap one. + +Unlike :ref:`MERGE ` and :ref:`CLUSTER `, +which aggregate intervals, ``DISJOIN`` *multiplies* rows: one target interval +becomes one or more sub-interval rows. The full target row passes through +unchanged and the sub-interval is appended as ``disjoin_chrom``, +``disjoin_start``, ``disjoin_end``. + +When no ``reference`` is given it defaults to the target set, so +``DISJOIN(features)`` splits the set against its own breakpoints. Selecting +the distinct sub-intervals then yields the globally non-overlapping partition +-- the equivalent of Bioconductor's ``GenomicRanges::disjoin()``. + +This is useful for: + +- Partitioning a set of intervals into non-overlapping segments +- Re-tiling target features onto a data-defined or external grid +- Splitting features so downstream aggregates never double-count overlaps + +Syntax +~~~~~~ + +.. code-block:: sql + + -- Self-mode: split the set against its own breakpoints + SELECT * FROM DISJOIN(features) + + -- Split target features against an explicit reference set + SELECT * FROM DISJOIN(features, reference := mask) + + -- The reference may be a subquery + SELECT * FROM DISJOIN(features, reference := (SELECT * FROM mask)) + +Parameters +~~~~~~~~~~ + +**target** + The table of intervals to split. + +**reference** *(optional)* + A table, CTE, or subquery whose interval boundaries supply the breakpoints. + Defaults to ``target`` when omitted. + +Return Value +~~~~~~~~~~~~ + +Every column of the matched target row, passed through unchanged, plus the +sub-interval: + +- ``disjoin_chrom`` - Chromosome of the sub-interval +- ``disjoin_start`` - Start of the sub-interval +- ``disjoin_end`` - End of the sub-interval + +A sub-interval that overlaps no reference interval is dropped (the coverage +filter). In self-mode every sub-interval is covered by its own parent, so +nothing is dropped. + +Examples +~~~~~~~~ + +**Partition a set of intervals:** + +Given two overlapping intervals ``A = [0, 20)`` and ``B = [10, 30)``, +``DISJOIN`` in self-mode cuts ``A`` at breakpoint ``10`` and ``B`` at +breakpoint ``20``: + +.. code-block:: sql + + SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end + FROM DISJOIN(features) + ORDER BY disjoin_start + + -- Returns the partition: [0,10), [10,20), [20,30) + +**Split features against a mask:** + +.. code-block:: sql + + SELECT name, disjoin_start, disjoin_end + FROM DISJOIN(features, reference := blacklist) + +**Re-tile against a uniform grid:** + +.. code-block:: sql + + WITH bins AS ( + SELECT 'chr1' AS chrom, x AS start, x + 1000 AS "end" + FROM range(0, 250000000, 1000) AS t(x) + ) + SELECT * FROM DISJOIN(features, reference := bins) + +.. note:: + + ``DISJOIN`` is a table function and appears in the ``FROM`` clause, like + ``NEAREST``. On PostgreSQL the derived table must be given an alias + (``FROM DISJOIN(features) AS d``). + +.. note:: + + The appended ``disjoin_start`` / ``disjoin_end`` columns are emitted in the + target table's coordinate system -- the same convention as its + passed-through ``start`` / ``end`` columns, so every column of an output row + shares one convention. Cut positions are computed canonically inside the + operator; only their final representation follows the target table. + +Related Operators +~~~~~~~~~~~~~~~~~ + +- :ref:`MERGE ` - Collapse overlapping intervals into one +- :ref:`CLUSTER ` - Assign cluster IDs without splitting diff --git a/docs/index.rst b/docs/index.rst index 91c98d9..718f0ab 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -62,6 +62,7 @@ See the following :doc:`recipes ` to learn how to use GIQL effect recipes/intersect recipes/distance recipes/clustering + recipes/disjoin recipes/advanced recipes/bedtools-migration diff --git a/docs/recipes/disjoin.rst b/docs/recipes/disjoin.rst new file mode 100644 index 0000000..775c398 --- /dev/null +++ b/docs/recipes/disjoin.rst @@ -0,0 +1,81 @@ +Disjoining Intervals +==================== + +This section covers patterns for splitting intervals at breakpoints using +GIQL's ``DISJOIN`` operator -- partitioning a set into non-overlapping +segments and re-tiling features against a reference grid. + +Partition a Set of Intervals +----------------------------- + +Build a Disjoint Partition +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Split a set of intervals into the maximal set of non-overlapping +sub-intervals defined by its own breakpoints: + +.. code-block:: sql + + SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end + FROM DISJOIN(features) + ORDER BY disjoin_chrom, disjoin_start + +**Use case:** Produce a non-overlapping segment track -- the equivalent of +Bioconductor's ``disjoin()`` -- that downstream queries can aggregate without +double-counting overlaps. + +Track Each Segment's Parent +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Keep the parent feature alongside each sub-interval: + +.. code-block:: sql + + SELECT name, disjoin_start, disjoin_end + FROM DISJOIN(features) + ORDER BY name, disjoin_start + +**Use case:** See how each original feature was fragmented and which segment +came from which parent. + +Split Against a Reference +------------------------- + +Split Features Against a Mask +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Cut target features at the boundaries of a reference (mask) set, keeping only +the pieces the mask covers: + +.. code-block:: sql + + SELECT name, disjoin_start, disjoin_end + FROM DISJOIN(features, reference := mask) + +**Use case:** Restrict features to mask regions while splitting them at mask +boundaries so no piece straddles a mask edge. + +Re-tile Against a Uniform Grid +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pass a generated set of fixed-width bins as the reference: + +.. code-block:: sql + + WITH bins AS ( + SELECT 'chr1' AS chrom, x AS start, x + 1000 AS "end" + FROM range(0, 250000000, 1000) AS t(x) + ) + SELECT * FROM DISJOIN(features, reference := bins) + +**Use case:** Break features onto a uniform coordinate grid so each piece +falls within a single bin. + +Coming from Bedtools? +--------------------- + +``DISJOIN`` has no single bedtools equivalent. The self-mode partition is +closest to the breakpoints implied by ``bedtools merge`` re-split at every +input boundary; splitting against a reference is closest to ``bedtools +intersect`` combined with the reference's boundaries. See the +:doc:`bedtools-migration` guide for related operations. diff --git a/docs/recipes/index.rst b/docs/recipes/index.rst index cc97e47..173ad53 100644 --- a/docs/recipes/index.rst +++ b/docs/recipes/index.rst @@ -19,6 +19,10 @@ Recipe Categories Clustering overlapping intervals, distance-based clustering, merging intervals, and aggregating cluster statistics. +:doc:`disjoin` + Splitting intervals at breakpoints, partitioning a set into + non-overlapping segments, and re-tiling features against a reference grid. + :doc:`advanced` Multi-range matching, complex filtering with joins, aggregate statistics, window expansions, and multi-table queries. From c895e75a6981a1e97aa2b70c21fff731c8ec02b1 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 11:16:08 -0400 Subject: [PATCH 005/142] test: Add operator usage-pattern matrix and functional engine suite Add tests/usage_patterns.py, a catalogue of the query contexts an operator appears in, and tests/test_usage_patterns.py, a functional suite that transpiles every DISJOIN usage pattern, executes it against a database-engine matrix (DuckDB now; SQLite and PostgreSQL are future entries), and snapshots the result rows with pytest-manifest. The catalogue is a per-operator-class descriptor framework. Only the table-function class is populated -- DISJOIN, exercised across 21 usage patterns in self- and reference-mode. AGGREGATE, PREDICATE, and SCALAR are defined extension points. Three patterns GIQL cannot transpile (a subquery, CTE, or nested operator in a table-function's target position) are catalogued as strict xfail. Add pytest-manifest as a dev dependency and register the usage marker. --- pyproject.toml | 2 + ..._executes_to_snapshot_across_profiles.yaml | 9892 +++++++++++++++++ tests/test_usage_patterns.py | 380 + tests/usage_patterns.py | 1051 ++ 4 files changed, 11325 insertions(+) create mode 100644 tests/manifests/test_usage_patterns.TestDisjoinUsageMatrix.test_disjoin_pattern_executes_to_snapshot_across_profiles.yaml create mode 100644 tests/test_usage_patterns.py create mode 100644 tests/usage_patterns.py diff --git a/pyproject.toml b/pyproject.toml index 8203b1b..ade33e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dev = [ "pytest>=7.0.0", "ruff>=0.1.0", "datafusion>=52.3.0", + "pytest-manifest>=1.0", ] docs = [ "sphinx>=7.0", @@ -59,6 +60,7 @@ path = "build-hooks/metadata.py" addopts = "--cov --cov-config=.coveragerc" markers = [ "integration: end-to-end tests that execute generated SQL against a real database", + "usage: the canonical query-usage pattern (giql.tests usage_patterns) a test exercises", ] [tool.ruff] diff --git a/tests/manifests/test_usage_patterns.TestDisjoinUsageMatrix.test_disjoin_pattern_executes_to_snapshot_across_profiles.yaml b/tests/manifests/test_usage_patterns.TestDisjoinUsageMatrix.test_disjoin_pattern_executes_to_snapshot_across_profiles.yaml new file mode 100644 index 0000000..7636dc3 --- /dev/null +++ b/tests/manifests/test_usage_patterns.TestDisjoinUsageMatrix.test_disjoin_pattern_executes_to_snapshot_across_profiles.yaml @@ -0,0 +1,9892 @@ +ADJACENT_INTERVALS-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 90 +ADJACENT_INTERVALS-reference-duckdb-ALIASED_PROJECTION: +- - 0 + - 30 +- - 30 + - 60 +- - 60 + - 90 +ADJACENT_INTERVALS-reference-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 30 +- - 60 +ADJACENT_INTERVALS-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-reference-duckdb-JOINED: +- - chr1 + - 0 + - 30 + - g1 +ADJACENT_INTERVALS-reference-duckdb-NESTED_AGGREGATE: +- - 3 + - 90 +ADJACENT_INTERVALS-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-reference-duckdb-OUTER_COUNT: +- - 3 +ADJACENT_INTERVALS-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 3 +ADJACENT_INTERVALS-reference-duckdb-OUTER_HAVING: +- - chr1 + - 3 +ADJACENT_INTERVALS-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 30 + - g1 +- - chr1 + - 30 + - 60 + - null +- - chr1 + - 60 + - 90 + - null +ADJACENT_INTERVALS-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +ADJACENT_INTERVALS-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 +ADJACENT_INTERVALS-reference-duckdb-OUTER_WHERE: [] +ADJACENT_INTERVALS-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 30 + - 0 +- - chr1 + - 30 + - 60 + - 0 +- - chr1 + - 60 + - 90 + - 0 +ADJACENT_INTERVALS-reference-duckdb-SET_DIFFERENCE: [] +ADJACENT_INTERVALS-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +ADJACENT_INTERVALS-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 + - 3 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 + - 3 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 60 + - 90 + - 3 +ADJACENT_INTERVALS-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 90 + - t + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 30 + - 60 +- - chr1 + - 0 + - 90 + - t + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 90 +ADJACENT_INTERVALS-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 30 +- - 30 + - 60 +- - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 30 +- - 60 +ADJACENT_INTERVALS-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 + - c + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 + - c + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-JOINED: +- - chr1 + - 0 + - 30 + - g1 +ADJACENT_INTERVALS-self-duckdb-NESTED_AGGREGATE: +- - 3 + - 90 +ADJACENT_INTERVALS-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 + - c + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-OUTER_COUNT: +- - 3 +ADJACENT_INTERVALS-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 3 +ADJACENT_INTERVALS-self-duckdb-OUTER_HAVING: +- - chr1 + - 3 +ADJACENT_INTERVALS-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 30 + - g1 +- - chr1 + - 30 + - 60 + - null +- - chr1 + - 60 + - 90 + - null +ADJACENT_INTERVALS-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +ADJACENT_INTERVALS-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 +ADJACENT_INTERVALS-self-duckdb-OUTER_WHERE: [] +ADJACENT_INTERVALS-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 30 + - 0 +- - chr1 + - 30 + - 60 + - 30 +- - chr1 + - 60 + - 90 + - 60 +ADJACENT_INTERVALS-self-duckdb-SET_DIFFERENCE: [] +ADJACENT_INTERVALS-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +ADJACENT_INTERVALS-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 + - c + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 + - c + - chr1 + - 60 + - 90 +ADJACENT_INTERVALS-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 + - 3 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 + - 3 +- - chr1 + - 60 + - 90 + - c + - chr1 + - 60 + - 90 + - 3 +ADJACENT_INTERVALS-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 30 + - a + - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 60 + - b + - chr1 + - 30 + - 60 +- - chr1 + - 60 + - 90 + - c + - chr1 + - 60 + - 90 +CANONICAL-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 180 +- - chr2 + - 30 +CANONICAL-reference-duckdb-ALIASED_PROJECTION: +- - 0 + - 70 +- - 10 + - 40 +- - 60 + - 70 +- - 90 + - 100 +- - 90 + - 180 +CANONICAL-reference-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 0 +- - 10 +- - 60 +- - 60 +- - 90 +- - 90 +- - 90 +- - 90 +CANONICAL-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-reference-duckdb-JOINED: +- - chr1 + - 0 + - 70 + - g1 +- - chr2 + - 10 + - 40 + - g2 +CANONICAL-reference-duckdb-NESTED_AGGREGATE: +- - 5 + - 210 +CANONICAL-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-reference-duckdb-OUTER_COUNT: +- - 5 +CANONICAL-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 70 +- - chr1 + - 60 + - 70 +- - chr1 + - 90 + - 100 +- - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 +CANONICAL-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +CANONICAL-reference-duckdb-OUTER_HAVING: +- - chr1 + - 4 +CANONICAL-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 70 + - g1 +- - chr1 + - 60 + - 70 + - null +- - chr1 + - 90 + - 100 + - null +- - chr1 + - 90 + - 180 + - null +- - chr2 + - 10 + - 40 + - g2 +CANONICAL-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +CANONICAL-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +CANONICAL-reference-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +CANONICAL-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 70 + - 0 +- - chr1 + - 0 + - 70 + - 60 +- - chr1 + - 60 + - 70 + - 0 +- - chr1 + - 60 + - 70 + - 60 +- - chr1 + - 90 + - 100 + - 0 +- - chr1 + - 90 + - 100 + - 60 +- - chr1 + - 90 + - 180 + - 0 +- - chr1 + - 90 + - 180 + - 60 +- - chr2 + - 10 + - 40 + - 10 +CANONICAL-reference-duckdb-SET_DIFFERENCE: [] +CANONICAL-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +CANONICAL-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 + - 4 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 + - 4 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 + - 1 +CANONICAL-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 220 +- - chr2 + - 30 +CANONICAL-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 60 +- - 10 + - 40 +- - 100 + - 180 +- - 60 + - 100 +- - 60 + - 100 +CANONICAL-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 100 +- - 10 +- - 60 +- - 60 +- - 60 +- - 60 +CANONICAL-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-self-duckdb-JOINED: +- - chr1 + - 0 + - 60 + - g1 +- - chr2 + - 10 + - 40 + - g2 +CANONICAL-self-duckdb-NESTED_AGGREGATE: +- - 5 + - 250 +CANONICAL-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-self-duckdb-OUTER_COUNT: +- - 5 +CANONICAL-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 60 +- - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 +CANONICAL-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +CANONICAL-self-duckdb-OUTER_HAVING: +- - chr1 + - 4 +CANONICAL-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 60 + - g1 +- - chr1 + - 100 + - 180 + - null +- - chr1 + - 60 + - 100 + - null +- - chr1 + - 60 + - 100 + - null +- - chr2 + - 10 + - 40 + - g2 +CANONICAL-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +CANONICAL-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +CANONICAL-self-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +CANONICAL-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 60 + - 0 +- - chr1 + - 100 + - 180 + - 60 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 60 +- - chr1 + - 60 + - 100 + - 60 +- - chr2 + - 10 + - 40 + - 10 +CANONICAL-self-duckdb-SET_DIFFERENCE: [] +CANONICAL-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +CANONICAL-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CANONICAL-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 + - 4 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 + - 4 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 + - 1 +CANONICAL-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 180 +- - chr2 + - 30 +CUSTOM_COLUMNS-reference-duckdb-ALIASED_PROJECTION: +- - 0 + - 70 +- - 10 + - 40 +- - 60 + - 70 +- - 90 + - 100 +- - 90 + - 180 +CUSTOM_COLUMNS-reference-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 0 +- - 10 +- - 60 +- - 60 +- - 90 +- - 90 +- - 90 +- - 90 +CUSTOM_COLUMNS-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-reference-duckdb-JOINED: +- - chr1 + - 0 + - 70 + - g1 +- - chr2 + - 10 + - 40 + - g2 +CUSTOM_COLUMNS-reference-duckdb-NESTED_AGGREGATE: +- - 5 + - 210 +CUSTOM_COLUMNS-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-reference-duckdb-OUTER_COUNT: +- - 5 +CUSTOM_COLUMNS-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 70 +- - chr1 + - 60 + - 70 +- - chr1 + - 90 + - 100 +- - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +CUSTOM_COLUMNS-reference-duckdb-OUTER_HAVING: +- - chr1 + - 4 +CUSTOM_COLUMNS-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 70 + - g1 +- - chr1 + - 60 + - 70 + - null +- - chr1 + - 90 + - 100 + - null +- - chr1 + - 90 + - 180 + - null +- - chr2 + - 10 + - 40 + - g2 +CUSTOM_COLUMNS-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +CUSTOM_COLUMNS-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +CUSTOM_COLUMNS-reference-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +CUSTOM_COLUMNS-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 70 + - 0 +- - chr1 + - 0 + - 70 + - 60 +- - chr1 + - 60 + - 70 + - 0 +- - chr1 + - 60 + - 70 + - 60 +- - chr1 + - 90 + - 100 + - 0 +- - chr1 + - 90 + - 100 + - 60 +- - chr1 + - 90 + - 180 + - 0 +- - chr1 + - 90 + - 180 + - 60 +- - chr2 + - 10 + - 40 + - 10 +CUSTOM_COLUMNS-reference-duckdb-SET_DIFFERENCE: [] +CUSTOM_COLUMNS-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +CUSTOM_COLUMNS-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 + - 4 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 + - 4 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 + - 1 +CUSTOM_COLUMNS-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 220 +- - chr2 + - 30 +CUSTOM_COLUMNS-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 60 +- - 10 + - 40 +- - 100 + - 180 +- - 60 + - 100 +- - 60 + - 100 +CUSTOM_COLUMNS-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 100 +- - 10 +- - 60 +- - 60 +- - 60 +- - 60 +CUSTOM_COLUMNS-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-self-duckdb-JOINED: +- - chr1 + - 0 + - 60 + - g1 +- - chr2 + - 10 + - 40 + - g2 +CUSTOM_COLUMNS-self-duckdb-NESTED_AGGREGATE: +- - 5 + - 250 +CUSTOM_COLUMNS-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-self-duckdb-OUTER_COUNT: +- - 5 +CUSTOM_COLUMNS-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 60 +- - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +CUSTOM_COLUMNS-self-duckdb-OUTER_HAVING: +- - chr1 + - 4 +CUSTOM_COLUMNS-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 60 + - g1 +- - chr1 + - 100 + - 180 + - null +- - chr1 + - 60 + - 100 + - null +- - chr1 + - 60 + - 100 + - null +- - chr2 + - 10 + - 40 + - g2 +CUSTOM_COLUMNS-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +CUSTOM_COLUMNS-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +CUSTOM_COLUMNS-self-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +CUSTOM_COLUMNS-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 60 + - 0 +- - chr1 + - 100 + - 180 + - 60 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 60 +- - chr1 + - 60 + - 100 + - 60 +- - chr2 + - 10 + - 40 + - 10 +CUSTOM_COLUMNS-self-duckdb-SET_DIFFERENCE: [] +CUSTOM_COLUMNS-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +CUSTOM_COLUMNS-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +CUSTOM_COLUMNS-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 + - 4 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 + - 4 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 + - 1 +CUSTOM_COLUMNS-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +DUPLICATE_TARGETS-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 60 +DUPLICATE_TARGETS-reference-duckdb-ALIASED_PROJECTION: +- - 40 + - 60 +- - 40 + - 60 +- - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-CHAINED_COOCCURRING: +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +DUPLICATE_TARGETS-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-JOINED: [] +DUPLICATE_TARGETS-reference-duckdb-NESTED_AGGREGATE: +- - 3 + - 60 +DUPLICATE_TARGETS-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-OUTER_COUNT: +- - 3 +DUPLICATE_TARGETS-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 3 +DUPLICATE_TARGETS-reference-duckdb-OUTER_HAVING: +- - chr1 + - 3 +DUPLICATE_TARGETS-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 40 + - 60 + - null +- - chr1 + - 40 + - 60 + - null +- - chr1 + - 40 + - 60 + - null +DUPLICATE_TARGETS-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-OUTER_WHERE: [] +DUPLICATE_TARGETS-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +DUPLICATE_TARGETS-reference-duckdb-SET_DIFFERENCE: [] +DUPLICATE_TARGETS-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +DUPLICATE_TARGETS-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 + - 3 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 + - 3 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 + - 3 +DUPLICATE_TARGETS-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 320 +DUPLICATE_TARGETS-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 40 +- - 0 + - 40 +- - 0 + - 40 +- - 40 + - 60 +- - 40 + - 60 +- - 40 + - 60 +- - 40 + - 60 +- - 60 + - 100 +- - 60 + - 100 +- - 60 + - 100 +DUPLICATE_TARGETS-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 0 +- - 0 +- - 0 +- - 0 +- - 0 +- - 0 +- - 0 +- - 0 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 60 +- - 60 +- - 60 +- - 60 +- - 60 +- - 60 +- - 60 +- - 60 +- - 60 +DUPLICATE_TARGETS-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 60 + - 100 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 60 + - 100 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-self-duckdb-JOINED: +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 0 + - 40 + - g1 +DUPLICATE_TARGETS-self-duckdb-NESTED_AGGREGATE: +- - 10 + - 320 +DUPLICATE_TARGETS-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 60 + - 100 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-self-duckdb-OUTER_COUNT: +- - 10 +DUPLICATE_TARGETS-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 40 +- - chr1 + - 40 + - 60 +- - chr1 + - 60 + - 100 +DUPLICATE_TARGETS-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 10 +DUPLICATE_TARGETS-self-duckdb-OUTER_HAVING: +- - chr1 + - 10 +DUPLICATE_TARGETS-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 40 + - 60 + - null +- - chr1 + - 40 + - 60 + - null +- - chr1 + - 40 + - 60 + - null +- - chr1 + - 40 + - 60 + - null +- - chr1 + - 60 + - 100 + - null +- - chr1 + - 60 + - 100 + - null +- - chr1 + - 60 + - 100 + - null +DUPLICATE_TARGETS-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +DUPLICATE_TARGETS-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-self-duckdb-OUTER_WHERE: [] +DUPLICATE_TARGETS-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 40 + - 60 + - 40 +- - chr1 + - 40 + - 60 + - 40 +- - chr1 + - 40 + - 60 + - 40 +- - chr1 + - 40 + - 60 + - 40 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +DUPLICATE_TARGETS-self-duckdb-SET_DIFFERENCE: [] +DUPLICATE_TARGETS-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +DUPLICATE_TARGETS-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 60 + - 100 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 60 + - 100 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 +DUPLICATE_TARGETS-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 + - 10 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 + - 10 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 60 + - 100 + - 10 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 + - 10 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 + - 10 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 60 + - 100 + - 10 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 + - 10 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 + - 10 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 60 + - 100 + - 10 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 + - 10 +DUPLICATE_TARGETS-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d1 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d2 + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - d3 + - chr1 + - 60 + - 100 +- - chr1 + - 40 + - 60 + - cut + - chr1 + - 40 + - 60 +EMPTY_REFERENCE-reference-duckdb-AGGREGATED_INTERVAL: [] +EMPTY_REFERENCE-reference-duckdb-ALIASED_PROJECTION: [] +EMPTY_REFERENCE-reference-duckdb-CHAINED_COOCCURRING: [] +EMPTY_REFERENCE-reference-duckdb-CHAINED_REFERENCE: [] +EMPTY_REFERENCE-reference-duckdb-DERIVED_SUBQUERY: [] +EMPTY_REFERENCE-reference-duckdb-JOINED: [] +EMPTY_REFERENCE-reference-duckdb-NESTED_AGGREGATE: +- - 0 + - null +EMPTY_REFERENCE-reference-duckdb-ORDERED_CTE: [] +EMPTY_REFERENCE-reference-duckdb-OUTER_COUNT: +- - 0 +EMPTY_REFERENCE-reference-duckdb-OUTER_DISTINCT: [] +EMPTY_REFERENCE-reference-duckdb-OUTER_GROUP_BY: [] +EMPTY_REFERENCE-reference-duckdb-OUTER_HAVING: [] +EMPTY_REFERENCE-reference-duckdb-OUTER_LEFT_JOIN: [] +EMPTY_REFERENCE-reference-duckdb-OUTER_ORDER_LIMIT: [] +EMPTY_REFERENCE-reference-duckdb-OUTER_SPATIAL_WHERE: [] +EMPTY_REFERENCE-reference-duckdb-OUTER_WHERE: [] +EMPTY_REFERENCE-reference-duckdb-SELF_JOIN_OVERLAP: [] +EMPTY_REFERENCE-reference-duckdb-SET_DIFFERENCE: [] +EMPTY_REFERENCE-reference-duckdb-SET_UNION: +- - chr1 +- - chr2 +EMPTY_REFERENCE-reference-duckdb-STANDALONE: [] +EMPTY_REFERENCE-reference-duckdb-SUBQUERY_REFERENCE: [] +EMPTY_REFERENCE-reference-duckdb-WINDOWED_PROJECTION: [] +EMPTY_REFERENCE-reference-duckdb-WRAPPING_CTE: [] +EMPTY_REFERENCE-self-duckdb-AGGREGATED_INTERVAL: [] +EMPTY_REFERENCE-self-duckdb-ALIASED_PROJECTION: [] +EMPTY_REFERENCE-self-duckdb-CHAINED_COOCCURRING: [] +EMPTY_REFERENCE-self-duckdb-CHAINED_REFERENCE: [] +EMPTY_REFERENCE-self-duckdb-DERIVED_SUBQUERY: [] +EMPTY_REFERENCE-self-duckdb-JOINED: [] +EMPTY_REFERENCE-self-duckdb-NESTED_AGGREGATE: +- - 0 + - null +EMPTY_REFERENCE-self-duckdb-ORDERED_CTE: [] +EMPTY_REFERENCE-self-duckdb-OUTER_COUNT: +- - 0 +EMPTY_REFERENCE-self-duckdb-OUTER_DISTINCT: [] +EMPTY_REFERENCE-self-duckdb-OUTER_GROUP_BY: [] +EMPTY_REFERENCE-self-duckdb-OUTER_HAVING: [] +EMPTY_REFERENCE-self-duckdb-OUTER_LEFT_JOIN: [] +EMPTY_REFERENCE-self-duckdb-OUTER_ORDER_LIMIT: [] +EMPTY_REFERENCE-self-duckdb-OUTER_SPATIAL_WHERE: [] +EMPTY_REFERENCE-self-duckdb-OUTER_WHERE: [] +EMPTY_REFERENCE-self-duckdb-SELF_JOIN_OVERLAP: [] +EMPTY_REFERENCE-self-duckdb-SET_DIFFERENCE: [] +EMPTY_REFERENCE-self-duckdb-SET_UNION: +- - chr1 +- - chr2 +EMPTY_REFERENCE-self-duckdb-STANDALONE: [] +EMPTY_REFERENCE-self-duckdb-SUBQUERY_REFERENCE: [] +EMPTY_REFERENCE-self-duckdb-WINDOWED_PROJECTION: [] +EMPTY_REFERENCE-self-duckdb-WRAPPING_CTE: [] +EMPTY_TARGET-reference-duckdb-AGGREGATED_INTERVAL: [] +EMPTY_TARGET-reference-duckdb-ALIASED_PROJECTION: [] +EMPTY_TARGET-reference-duckdb-CHAINED_COOCCURRING: [] +EMPTY_TARGET-reference-duckdb-CHAINED_REFERENCE: [] +EMPTY_TARGET-reference-duckdb-DERIVED_SUBQUERY: [] +EMPTY_TARGET-reference-duckdb-JOINED: [] +EMPTY_TARGET-reference-duckdb-NESTED_AGGREGATE: +- - 0 + - null +EMPTY_TARGET-reference-duckdb-ORDERED_CTE: [] +EMPTY_TARGET-reference-duckdb-OUTER_COUNT: +- - 0 +EMPTY_TARGET-reference-duckdb-OUTER_DISTINCT: [] +EMPTY_TARGET-reference-duckdb-OUTER_GROUP_BY: [] +EMPTY_TARGET-reference-duckdb-OUTER_HAVING: [] +EMPTY_TARGET-reference-duckdb-OUTER_LEFT_JOIN: [] +EMPTY_TARGET-reference-duckdb-OUTER_ORDER_LIMIT: [] +EMPTY_TARGET-reference-duckdb-OUTER_SPATIAL_WHERE: [] +EMPTY_TARGET-reference-duckdb-OUTER_WHERE: [] +EMPTY_TARGET-reference-duckdb-SELF_JOIN_OVERLAP: [] +EMPTY_TARGET-reference-duckdb-SET_DIFFERENCE: [] +EMPTY_TARGET-reference-duckdb-SET_UNION: +- - chr1 +- - chr2 +EMPTY_TARGET-reference-duckdb-STANDALONE: [] +EMPTY_TARGET-reference-duckdb-SUBQUERY_REFERENCE: [] +EMPTY_TARGET-reference-duckdb-WINDOWED_PROJECTION: [] +EMPTY_TARGET-reference-duckdb-WRAPPING_CTE: [] +EMPTY_TARGET-self-duckdb-AGGREGATED_INTERVAL: [] +EMPTY_TARGET-self-duckdb-ALIASED_PROJECTION: [] +EMPTY_TARGET-self-duckdb-CHAINED_COOCCURRING: [] +EMPTY_TARGET-self-duckdb-CHAINED_REFERENCE: [] +EMPTY_TARGET-self-duckdb-DERIVED_SUBQUERY: [] +EMPTY_TARGET-self-duckdb-JOINED: [] +EMPTY_TARGET-self-duckdb-NESTED_AGGREGATE: +- - 0 + - null +EMPTY_TARGET-self-duckdb-ORDERED_CTE: [] +EMPTY_TARGET-self-duckdb-OUTER_COUNT: +- - 0 +EMPTY_TARGET-self-duckdb-OUTER_DISTINCT: [] +EMPTY_TARGET-self-duckdb-OUTER_GROUP_BY: [] +EMPTY_TARGET-self-duckdb-OUTER_HAVING: [] +EMPTY_TARGET-self-duckdb-OUTER_LEFT_JOIN: [] +EMPTY_TARGET-self-duckdb-OUTER_ORDER_LIMIT: [] +EMPTY_TARGET-self-duckdb-OUTER_SPATIAL_WHERE: [] +EMPTY_TARGET-self-duckdb-OUTER_WHERE: [] +EMPTY_TARGET-self-duckdb-SELF_JOIN_OVERLAP: [] +EMPTY_TARGET-self-duckdb-SET_DIFFERENCE: [] +EMPTY_TARGET-self-duckdb-SET_UNION: +- - chr1 +- - chr2 +EMPTY_TARGET-self-duckdb-STANDALONE: [] +EMPTY_TARGET-self-duckdb-SUBQUERY_REFERENCE: [] +EMPTY_TARGET-self-duckdb-WINDOWED_PROJECTION: [] +EMPTY_TARGET-self-duckdb-WRAPPING_CTE: [] +IDENTICAL_INTERVALS-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 60 +IDENTICAL_INTERVALS-reference-duckdb-ALIASED_PROJECTION: +- - 20 + - 50 +- - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-CHAINED_COOCCURRING: +- - 20 +- - 20 +- - 20 +- - 20 +IDENTICAL_INTERVALS-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-JOINED: +- - chr1 + - 20 + - 50 + - g1 +- - chr1 + - 20 + - 50 + - g1 +IDENTICAL_INTERVALS-reference-duckdb-NESTED_AGGREGATE: +- - 2 + - 60 +IDENTICAL_INTERVALS-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_COUNT: +- - 2 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 2 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_HAVING: +- - chr1 + - 2 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 20 + - 50 + - g1 +- - chr1 + - 20 + - 50 + - g1 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-OUTER_WHERE: [] +IDENTICAL_INTERVALS-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +IDENTICAL_INTERVALS-reference-duckdb-SET_DIFFERENCE: [] +IDENTICAL_INTERVALS-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr2 +IDENTICAL_INTERVALS-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 + - 2 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 + - 2 +IDENTICAL_INTERVALS-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 160 +IDENTICAL_INTERVALS-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 20 +- - 0 + - 20 +- - 20 + - 50 +- - 20 + - 50 +- - 20 + - 50 +- - 50 + - 80 +IDENTICAL_INTERVALS-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 0 +- - 0 +- - 0 +- - 20 +- - 20 +- - 20 +- - 20 +- - 20 +- - 20 +- - 20 +- - 20 +- - 20 +- - 50 +IDENTICAL_INTERVALS-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 50 + - 80 +IDENTICAL_INTERVALS-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 50 + - 80 +IDENTICAL_INTERVALS-self-duckdb-JOINED: +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 20 + - 50 + - g1 +- - chr1 + - 20 + - 50 + - g1 +- - chr1 + - 20 + - 50 + - g1 +IDENTICAL_INTERVALS-self-duckdb-NESTED_AGGREGATE: +- - 6 + - 160 +IDENTICAL_INTERVALS-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 50 + - 80 +IDENTICAL_INTERVALS-self-duckdb-OUTER_COUNT: +- - 6 +IDENTICAL_INTERVALS-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 20 +- - chr1 + - 20 + - 50 +- - chr1 + - 50 + - 80 +IDENTICAL_INTERVALS-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 6 +IDENTICAL_INTERVALS-self-duckdb-OUTER_HAVING: +- - chr1 + - 6 +IDENTICAL_INTERVALS-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 20 + - 50 + - g1 +- - chr1 + - 20 + - 50 + - g1 +- - chr1 + - 20 + - 50 + - g1 +- - chr1 + - 50 + - 80 + - null +IDENTICAL_INTERVALS-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 +IDENTICAL_INTERVALS-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 +IDENTICAL_INTERVALS-self-duckdb-OUTER_WHERE: [] +IDENTICAL_INTERVALS-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 20 + - 0 +- - chr1 + - 0 + - 20 + - 0 +- - chr1 + - 0 + - 20 + - 0 +- - chr1 + - 0 + - 20 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 0 +- - chr1 + - 20 + - 50 + - 20 +- - chr1 + - 20 + - 50 + - 20 +- - chr1 + - 20 + - 50 + - 20 +- - chr1 + - 50 + - 80 + - 20 +IDENTICAL_INTERVALS-self-duckdb-SET_DIFFERENCE: [] +IDENTICAL_INTERVALS-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +IDENTICAL_INTERVALS-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 50 + - 80 +IDENTICAL_INTERVALS-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 50 + - 80 +IDENTICAL_INTERVALS-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 + - 6 +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 + - 6 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 0 + - 20 + - 6 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 + - 6 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 + - 6 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 50 + - 80 + - 6 +IDENTICAL_INTERVALS-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x1 + - chr1 + - 20 + - 50 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 50 + - x2 + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 20 + - 50 +- - chr1 + - 20 + - 80 + - y + - chr1 + - 50 + - 80 +MANY_CHROMOSOMES-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 50 +- - chr2 + - 30 +- - chr3 + - 30 +- - chrX + - 20 +MANY_CHROMOSOMES-reference-duckdb-ALIASED_PROJECTION: +- - 10 + - 40 +- - 30 + - 60 +- - 5 + - 25 +- - 50 + - 100 +MANY_CHROMOSOMES-reference-duckdb-CHAINED_COOCCURRING: +- - 10 +- - 30 +- - 50 +- - 5 +MANY_CHROMOSOMES-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-reference-duckdb-JOINED: +- - chr1 + - 50 + - 100 + - g1 +- - chr2 + - 30 + - 60 + - g2 +- - chr3 + - 10 + - 40 + - g3 +- - chrX + - 5 + - 25 + - g4 +MANY_CHROMOSOMES-reference-duckdb-NESTED_AGGREGATE: +- - 4 + - 130 +MANY_CHROMOSOMES-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-reference-duckdb-OUTER_COUNT: +- - 4 +MANY_CHROMOSOMES-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 50 + - 100 +- - chr2 + - 30 + - 60 +- - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 +MANY_CHROMOSOMES-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 1 +- - chr2 + - 1 +- - chr3 + - 1 +- - chrX + - 1 +MANY_CHROMOSOMES-reference-duckdb-OUTER_HAVING: [] +MANY_CHROMOSOMES-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 50 + - 100 + - g1 +- - chr2 + - 30 + - 60 + - g2 +- - chr3 + - 10 + - 40 + - g3 +- - chrX + - 5 + - 25 + - g4 +MANY_CHROMOSOMES-reference-duckdb-OUTER_ORDER_LIMIT: +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-reference-duckdb-OUTER_SPATIAL_WHERE: [] +MANY_CHROMOSOMES-reference-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +MANY_CHROMOSOMES-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 50 + - 100 + - 0 +- - chr2 + - 30 + - 60 + - 0 +- - chr3 + - 10 + - 40 + - 10 +- - chrX + - 5 + - 25 + - 5 +MANY_CHROMOSOMES-reference-duckdb-SET_DIFFERENCE: [] +MANY_CHROMOSOMES-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr2 +- - chr2 +- - chr3 +- - chr3 +- - chrX +- - chrX +MANY_CHROMOSOMES-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 + - 1 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 + - 1 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 + - 1 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 + - 1 +MANY_CHROMOSOMES-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 200 +- - chr2 + - 120 +- - chr3 + - 30 +- - chrX + - 20 +MANY_CHROMOSOMES-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 30 +- - 0 + - 50 +- - 10 + - 40 +- - 100 + - 150 +- - 30 + - 60 +- - 30 + - 60 +- - 5 + - 25 +- - 50 + - 100 +- - 50 + - 100 +- - 60 + - 90 +MANY_CHROMOSOMES-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 0 +- - 100 +- - 10 +- - 30 +- - 30 +- - 30 +- - 30 +- - 50 +- - 50 +- - 50 +- - 50 +- - 5 +- - 60 +MANY_CHROMOSOMES-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 0 + - 30 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 60 + - 90 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 0 + - 30 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 60 + - 90 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-self-duckdb-JOINED: +- - chr1 + - 0 + - 50 + - g1 +- - chr1 + - 100 + - 150 + - g1 +- - chr1 + - 50 + - 100 + - g1 +- - chr1 + - 50 + - 100 + - g1 +- - chr2 + - 0 + - 30 + - g2 +- - chr2 + - 30 + - 60 + - g2 +- - chr2 + - 30 + - 60 + - g2 +- - chr2 + - 60 + - 90 + - g2 +- - chr3 + - 10 + - 40 + - g3 +- - chrX + - 5 + - 25 + - g4 +MANY_CHROMOSOMES-self-duckdb-NESTED_AGGREGATE: +- - 10 + - 370 +MANY_CHROMOSOMES-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 0 + - 30 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 60 + - 90 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-self-duckdb-OUTER_COUNT: +- - 10 +MANY_CHROMOSOMES-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 50 +- - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 30 +- - chr2 + - 30 + - 60 +- - chr2 + - 60 + - 90 +- - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 +MANY_CHROMOSOMES-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 4 +- - chr3 + - 1 +- - chrX + - 1 +MANY_CHROMOSOMES-self-duckdb-OUTER_HAVING: +- - chr1 + - 4 +- - chr2 + - 4 +MANY_CHROMOSOMES-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 50 + - g1 +- - chr1 + - 100 + - 150 + - g1 +- - chr1 + - 50 + - 100 + - g1 +- - chr1 + - 50 + - 100 + - g1 +- - chr2 + - 0 + - 30 + - g2 +- - chr2 + - 30 + - 60 + - g2 +- - chr2 + - 30 + - 60 + - g2 +- - chr2 + - 60 + - 90 + - g2 +- - chr3 + - 10 + - 40 + - g3 +- - chrX + - 5 + - 25 + - g4 +MANY_CHROMOSOMES-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +MANY_CHROMOSOMES-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +MANY_CHROMOSOMES-self-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 +MANY_CHROMOSOMES-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 50 + - 0 +- - chr1 + - 100 + - 150 + - 50 +- - chr1 + - 50 + - 100 + - 0 +- - chr1 + - 50 + - 100 + - 0 +- - chr1 + - 50 + - 100 + - 50 +- - chr1 + - 50 + - 100 + - 50 +- - chr2 + - 0 + - 30 + - 0 +- - chr2 + - 30 + - 60 + - 0 +- - chr2 + - 30 + - 60 + - 0 +- - chr2 + - 30 + - 60 + - 30 +- - chr2 + - 30 + - 60 + - 30 +- - chr2 + - 60 + - 90 + - 30 +- - chr3 + - 10 + - 40 + - 10 +- - chrX + - 5 + - 25 + - 5 +MANY_CHROMOSOMES-self-duckdb-SET_DIFFERENCE: [] +MANY_CHROMOSOMES-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +- - chr2 +- - chr2 +- - chr2 +- - chr3 +- - chr3 +- - chrX +- - chrX +MANY_CHROMOSOMES-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 0 + - 30 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 60 + - 90 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 0 + - 30 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 60 + - 90 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MANY_CHROMOSOMES-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 + - 4 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 + - 4 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 + - 4 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 + - 4 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 0 + - 30 + - 4 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 + - 4 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 30 + - 60 + - 4 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 60 + - 90 + - 4 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 + - 1 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 + - 1 +MANY_CHROMOSOMES-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 50 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 50 + - 100 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 100 + - 150 +- - chr1 + - 50 + - 150 + - b + - chr1 + - 50 + - 100 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 0 + - 30 +- - chr2 + - 0 + - 60 + - c + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 30 + - 60 +- - chr2 + - 30 + - 90 + - d + - chr2 + - 60 + - 90 +- - chr3 + - 10 + - 40 + - e + - chr3 + - 10 + - 40 +- - chrX + - 5 + - 25 + - f + - chrX + - 5 + - 25 +MIXED_ENCODING-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 180 +- - chr2 + - 30 +MIXED_ENCODING-reference-duckdb-ALIASED_PROJECTION: +- - 0 + - 70 +- - 10 + - 40 +- - 60 + - 70 +- - 90 + - 100 +- - 90 + - 180 +MIXED_ENCODING-reference-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 0 +- - 10 +- - 60 +- - 60 +- - 90 +- - 90 +- - 90 +- - 90 +MIXED_ENCODING-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 1 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 91 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 91 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-reference-duckdb-JOINED: +- - chr1 + - 0 + - 70 + - g1 +- - chr2 + - 10 + - 40 + - g2 +MIXED_ENCODING-reference-duckdb-NESTED_AGGREGATE: +- - 5 + - 210 +MIXED_ENCODING-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-reference-duckdb-OUTER_COUNT: +- - 5 +MIXED_ENCODING-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 70 +- - chr1 + - 60 + - 70 +- - chr1 + - 90 + - 100 +- - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 +MIXED_ENCODING-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +MIXED_ENCODING-reference-duckdb-OUTER_HAVING: +- - chr1 + - 4 +MIXED_ENCODING-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 70 + - g1 +- - chr1 + - 60 + - 70 + - null +- - chr1 + - 90 + - 100 + - null +- - chr1 + - 90 + - 180 + - null +- - chr2 + - 10 + - 40 + - g2 +MIXED_ENCODING-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +MIXED_ENCODING-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +MIXED_ENCODING-reference-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +MIXED_ENCODING-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 70 + - 0 +- - chr1 + - 0 + - 70 + - 60 +- - chr1 + - 60 + - 70 + - 0 +- - chr1 + - 60 + - 70 + - 60 +- - chr1 + - 90 + - 100 + - 0 +- - chr1 + - 90 + - 100 + - 60 +- - chr1 + - 90 + - 180 + - 0 +- - chr1 + - 90 + - 180 + - 60 +- - chr2 + - 10 + - 40 + - 10 +MIXED_ENCODING-reference-duckdb-SET_DIFFERENCE: [] +MIXED_ENCODING-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +MIXED_ENCODING-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 1 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 91 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 91 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 + - 4 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 + - 4 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 + - 1 +MIXED_ENCODING-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 70 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 90 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 70 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 90 + - 180 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 220 +- - chr2 + - 30 +MIXED_ENCODING-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 60 +- - 10 + - 40 +- - 100 + - 180 +- - 60 + - 100 +- - 60 + - 100 +MIXED_ENCODING-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 100 +- - 10 +- - 60 +- - 60 +- - 60 +- - 60 +MIXED_ENCODING-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-self-duckdb-JOINED: +- - chr1 + - 0 + - 60 + - g1 +- - chr2 + - 10 + - 40 + - g2 +MIXED_ENCODING-self-duckdb-NESTED_AGGREGATE: +- - 5 + - 250 +MIXED_ENCODING-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-self-duckdb-OUTER_COUNT: +- - 5 +MIXED_ENCODING-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 60 +- - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 +MIXED_ENCODING-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +MIXED_ENCODING-self-duckdb-OUTER_HAVING: +- - chr1 + - 4 +MIXED_ENCODING-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 60 + - g1 +- - chr1 + - 100 + - 180 + - null +- - chr1 + - 60 + - 100 + - null +- - chr1 + - 60 + - 100 + - null +- - chr2 + - 10 + - 40 + - g2 +MIXED_ENCODING-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +MIXED_ENCODING-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +MIXED_ENCODING-self-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +MIXED_ENCODING-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 60 + - 0 +- - chr1 + - 100 + - 180 + - 60 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 60 +- - chr1 + - 60 + - 100 + - 60 +- - chr2 + - 10 + - 40 + - 10 +MIXED_ENCODING-self-duckdb-SET_DIFFERENCE: [] +MIXED_ENCODING-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +MIXED_ENCODING-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +MIXED_ENCODING-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 + - 4 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 + - 4 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 + - 4 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 + - 1 +MIXED_ENCODING-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - a + - chr1 + - 0 + - 60 +- - chr1 + - 0 + - 100 + - a + - chr1 + - 60 + - 100 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 100 + - 180 +- - chr1 + - 60 + - 180 + - b + - chr1 + - 60 + - 100 +- - chr2 + - 10 + - 40 + - c + - chr2 + - 10 + - 40 +NESTED_INTERVALS-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 30 +NESTED_INTERVALS-reference-duckdb-ALIASED_PROJECTION: +- - 30 + - 40 +- - 40 + - 50 +- - 50 + - 60 +NESTED_INTERVALS-reference-duckdb-CHAINED_COOCCURRING: +- - 30 +- - 40 +- - 50 +NESTED_INTERVALS-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +NESTED_INTERVALS-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +NESTED_INTERVALS-reference-duckdb-JOINED: [] +NESTED_INTERVALS-reference-duckdb-NESTED_AGGREGATE: +- - 3 + - 30 +NESTED_INTERVALS-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +NESTED_INTERVALS-reference-duckdb-OUTER_COUNT: +- - 3 +NESTED_INTERVALS-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 30 + - 40 +- - chr1 + - 40 + - 50 +- - chr1 + - 50 + - 60 +NESTED_INTERVALS-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 3 +NESTED_INTERVALS-reference-duckdb-OUTER_HAVING: +- - chr1 + - 3 +NESTED_INTERVALS-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 30 + - 40 + - null +- - chr1 + - 40 + - 50 + - null +- - chr1 + - 50 + - 60 + - null +NESTED_INTERVALS-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +NESTED_INTERVALS-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +NESTED_INTERVALS-reference-duckdb-OUTER_WHERE: [] +NESTED_INTERVALS-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 30 + - 40 + - 0 +- - chr1 + - 40 + - 50 + - 0 +- - chr1 + - 50 + - 60 + - 0 +NESTED_INTERVALS-reference-duckdb-SET_DIFFERENCE: [] +NESTED_INTERVALS-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +NESTED_INTERVALS-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +NESTED_INTERVALS-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +NESTED_INTERVALS-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 + - 3 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 + - 3 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 + - 3 +NESTED_INTERVALS-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +NESTED_INTERVALS-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 140 +NESTED_INTERVALS-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 30 +- - 30 + - 40 +- - 30 + - 40 +- - 40 + - 50 +- - 40 + - 50 +- - 40 + - 50 +- - 50 + - 60 +- - 50 + - 60 +- - 60 + - 100 +NESTED_INTERVALS-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 30 +- - 30 +- - 30 +- - 30 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 50 +- - 50 +- - 50 +- - 50 +- - 60 +NESTED_INTERVALS-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 60 + - 100 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 50 + - 60 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 +NESTED_INTERVALS-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 60 + - 100 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 50 + - 60 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 +NESTED_INTERVALS-self-duckdb-JOINED: +- - chr1 + - 0 + - 30 + - g1 +NESTED_INTERVALS-self-duckdb-NESTED_AGGREGATE: +- - 9 + - 140 +NESTED_INTERVALS-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 60 + - 100 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 50 + - 60 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 +NESTED_INTERVALS-self-duckdb-OUTER_COUNT: +- - 9 +NESTED_INTERVALS-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 30 +- - chr1 + - 30 + - 40 +- - chr1 + - 40 + - 50 +- - chr1 + - 50 + - 60 +- - chr1 + - 60 + - 100 +NESTED_INTERVALS-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 9 +NESTED_INTERVALS-self-duckdb-OUTER_HAVING: +- - chr1 + - 9 +NESTED_INTERVALS-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 30 + - g1 +- - chr1 + - 30 + - 40 + - null +- - chr1 + - 30 + - 40 + - null +- - chr1 + - 40 + - 50 + - null +- - chr1 + - 40 + - 50 + - null +- - chr1 + - 40 + - 50 + - null +- - chr1 + - 50 + - 60 + - null +- - chr1 + - 50 + - 60 + - null +- - chr1 + - 60 + - 100 + - null +NESTED_INTERVALS-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +NESTED_INTERVALS-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 +NESTED_INTERVALS-self-duckdb-OUTER_WHERE: [] +NESTED_INTERVALS-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 30 + - 0 +- - chr1 + - 30 + - 40 + - 0 +- - chr1 + - 30 + - 40 + - 0 +- - chr1 + - 30 + - 40 + - 30 +- - chr1 + - 30 + - 40 + - 30 +- - chr1 + - 40 + - 50 + - 0 +- - chr1 + - 40 + - 50 + - 0 +- - chr1 + - 40 + - 50 + - 0 +- - chr1 + - 40 + - 50 + - 30 +- - chr1 + - 40 + - 50 + - 30 +- - chr1 + - 40 + - 50 + - 30 +- - chr1 + - 40 + - 50 + - 40 +- - chr1 + - 40 + - 50 + - 40 +- - chr1 + - 40 + - 50 + - 40 +- - chr1 + - 50 + - 60 + - 0 +- - chr1 + - 50 + - 60 + - 0 +- - chr1 + - 50 + - 60 + - 30 +- - chr1 + - 50 + - 60 + - 30 +- - chr1 + - 60 + - 100 + - 0 +NESTED_INTERVALS-self-duckdb-SET_DIFFERENCE: [] +NESTED_INTERVALS-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +NESTED_INTERVALS-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 60 + - 100 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 50 + - 60 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 +NESTED_INTERVALS-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 60 + - 100 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 50 + - 60 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 +NESTED_INTERVALS-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 + - 9 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 + - 9 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 + - 9 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 + - 9 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 60 + - 100 + - 9 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 + - 9 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 + - 9 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 50 + - 60 + - 9 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 + - 9 +NESTED_INTERVALS-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 0 + - 30 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 30 + - 40 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 40 + - 50 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 50 + - 60 +- - chr1 + - 0 + - 100 + - outer + - chr1 + - 60 + - 100 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 30 + - 40 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 40 + - 50 +- - chr1 + - 30 + - 60 + - inner + - chr1 + - 50 + - 60 +- - chr1 + - 40 + - 50 + - innermost + - chr1 + - 40 + - 50 +ONE_BASED_CLOSED_TARGET-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 176 +- - chr2 + - 29 +ONE_BASED_CLOSED_TARGET-reference-duckdb-ALIASED_PROJECTION: +- - 1 + - 70 +- - 11 + - 40 +- - 61 + - 70 +- - 91 + - 100 +- - 91 + - 180 +ONE_BASED_CLOSED_TARGET-reference-duckdb-CHAINED_COOCCURRING: +- - 11 +- - 1 +- - 1 +- - 61 +- - 61 +- - 91 +- - 91 +- - 91 +- - 91 +ONE_BASED_CLOSED_TARGET-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 2 + - 70 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 92 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 70 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 92 + - 180 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 91 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 70 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 91 + - 180 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-reference-duckdb-JOINED: +- - chr1 + - 1 + - 70 + - g1 +- - chr2 + - 11 + - 40 + - g2 +ONE_BASED_CLOSED_TARGET-reference-duckdb-NESTED_AGGREGATE: +- - 5 + - 205 +ONE_BASED_CLOSED_TARGET-reference-duckdb-ORDERED_CTE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 91 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 70 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 91 + - 180 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_COUNT: +- - 5 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 1 + - 70 +- - chr1 + - 61 + - 70 +- - chr1 + - 91 + - 100 +- - chr1 + - 91 + - 180 +- - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_HAVING: +- - chr1 + - 4 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 1 + - 70 + - g1 +- - chr1 + - 61 + - 70 + - null +- - chr1 + - 91 + - 100 + - null +- - chr1 + - 91 + - 180 + - null +- - chr2 + - 11 + - 40 + - g2 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 +ONE_BASED_CLOSED_TARGET-reference-duckdb-OUTER_WHERE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 91 + - 180 +ONE_BASED_CLOSED_TARGET-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 1 + - 70 + - 1 +- - chr1 + - 1 + - 70 + - 61 +- - chr1 + - 61 + - 70 + - 1 +- - chr1 + - 61 + - 70 + - 61 +- - chr1 + - 91 + - 100 + - 1 +- - chr1 + - 91 + - 100 + - 61 +- - chr1 + - 91 + - 180 + - 1 +- - chr1 + - 91 + - 180 + - 61 +- - chr2 + - 11 + - 40 + - 11 +ONE_BASED_CLOSED_TARGET-reference-duckdb-SET_DIFFERENCE: [] +ONE_BASED_CLOSED_TARGET-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +ONE_BASED_CLOSED_TARGET-reference-duckdb-STANDALONE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 91 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 70 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 91 + - 180 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 2 + - 70 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 92 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 70 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 92 + - 180 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 + - 4 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 91 + - 100 + - 4 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 70 + - 4 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 91 + - 180 + - 4 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 + - 1 +ONE_BASED_CLOSED_TARGET-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 70 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 91 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 70 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 91 + - 180 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 216 +- - chr2 + - 29 +ONE_BASED_CLOSED_TARGET-self-duckdb-ALIASED_PROJECTION: +- - 1 + - 60 +- - 101 + - 180 +- - 11 + - 40 +- - 61 + - 100 +- - 61 + - 100 +ONE_BASED_CLOSED_TARGET-self-duckdb-CHAINED_COOCCURRING: +- - 101 +- - 11 +- - 1 +- - 61 +- - 61 +- - 61 +- - 61 +ONE_BASED_CLOSED_TARGET-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 2 + - 60 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 62 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 102 + - 180 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 62 + - 100 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 12 + - 40 +ONE_BASED_CLOSED_TARGET-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 61 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 101 + - 180 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 100 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-self-duckdb-JOINED: +- - chr1 + - 1 + - 60 + - g1 +- - chr2 + - 11 + - 40 + - g2 +ONE_BASED_CLOSED_TARGET-self-duckdb-NESTED_AGGREGATE: +- - 5 + - 245 +ONE_BASED_CLOSED_TARGET-self-duckdb-ORDERED_CTE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 61 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 101 + - 180 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 100 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_COUNT: +- - 5 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 1 + - 60 +- - chr1 + - 101 + - 180 +- - chr1 + - 61 + - 100 +- - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +- - chr2 + - 1 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_HAVING: +- - chr1 + - 4 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 1 + - 60 + - g1 +- - chr1 + - 101 + - 180 + - null +- - chr1 + - 61 + - 100 + - null +- - chr1 + - 61 + - 100 + - null +- - chr2 + - 11 + - 40 + - g2 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 +ONE_BASED_CLOSED_TARGET-self-duckdb-OUTER_WHERE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 101 + - 180 +ONE_BASED_CLOSED_TARGET-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 1 + - 60 + - 1 +- - chr1 + - 101 + - 180 + - 61 +- - chr1 + - 61 + - 100 + - 1 +- - chr1 + - 61 + - 100 + - 1 +- - chr1 + - 61 + - 100 + - 61 +- - chr1 + - 61 + - 100 + - 61 +- - chr2 + - 11 + - 40 + - 11 +ONE_BASED_CLOSED_TARGET-self-duckdb-SET_DIFFERENCE: [] +ONE_BASED_CLOSED_TARGET-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +- - chr2 +ONE_BASED_CLOSED_TARGET-self-duckdb-STANDALONE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 61 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 101 + - 180 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 100 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +ONE_BASED_CLOSED_TARGET-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 2 + - 61 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 62 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 101 + - 180 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 61 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 62 + - 100 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 12 + - 40 +ONE_BASED_CLOSED_TARGET-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 + - 4 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 61 + - 100 + - 4 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 101 + - 180 + - 4 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 100 + - 4 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 + - 1 +ONE_BASED_CLOSED_TARGET-self-duckdb-WRAPPING_CTE: +- - chr1 + - 1 + - 100 + - a + - chr1 + - 1 + - 60 +- - chr1 + - 1 + - 100 + - a + - chr1 + - 61 + - 100 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 101 + - 180 +- - chr1 + - 61 + - 180 + - b + - chr1 + - 61 + - 100 +- - chr2 + - 11 + - 40 + - c + - chr2 + - 11 + - 40 +POINT_INTERVALS-reference-duckdb-AGGREGATED_INTERVAL: [] +POINT_INTERVALS-reference-duckdb-ALIASED_PROJECTION: [] +POINT_INTERVALS-reference-duckdb-CHAINED_COOCCURRING: [] +POINT_INTERVALS-reference-duckdb-CHAINED_REFERENCE: [] +POINT_INTERVALS-reference-duckdb-DERIVED_SUBQUERY: [] +POINT_INTERVALS-reference-duckdb-JOINED: [] +POINT_INTERVALS-reference-duckdb-NESTED_AGGREGATE: +- - 0 + - null +POINT_INTERVALS-reference-duckdb-ORDERED_CTE: [] +POINT_INTERVALS-reference-duckdb-OUTER_COUNT: +- - 0 +POINT_INTERVALS-reference-duckdb-OUTER_DISTINCT: [] +POINT_INTERVALS-reference-duckdb-OUTER_GROUP_BY: [] +POINT_INTERVALS-reference-duckdb-OUTER_HAVING: [] +POINT_INTERVALS-reference-duckdb-OUTER_LEFT_JOIN: [] +POINT_INTERVALS-reference-duckdb-OUTER_ORDER_LIMIT: [] +POINT_INTERVALS-reference-duckdb-OUTER_SPATIAL_WHERE: [] +POINT_INTERVALS-reference-duckdb-OUTER_WHERE: [] +POINT_INTERVALS-reference-duckdb-SELF_JOIN_OVERLAP: [] +POINT_INTERVALS-reference-duckdb-SET_DIFFERENCE: [] +POINT_INTERVALS-reference-duckdb-SET_UNION: +- - chr1 +- - chr2 +POINT_INTERVALS-reference-duckdb-STANDALONE: [] +POINT_INTERVALS-reference-duckdb-SUBQUERY_REFERENCE: [] +POINT_INTERVALS-reference-duckdb-WINDOWED_PROJECTION: [] +POINT_INTERVALS-reference-duckdb-WRAPPING_CTE: [] +POINT_INTERVALS-self-duckdb-AGGREGATED_INTERVAL: [] +POINT_INTERVALS-self-duckdb-ALIASED_PROJECTION: [] +POINT_INTERVALS-self-duckdb-CHAINED_COOCCURRING: [] +POINT_INTERVALS-self-duckdb-CHAINED_REFERENCE: [] +POINT_INTERVALS-self-duckdb-DERIVED_SUBQUERY: [] +POINT_INTERVALS-self-duckdb-JOINED: [] +POINT_INTERVALS-self-duckdb-NESTED_AGGREGATE: +- - 0 + - null +POINT_INTERVALS-self-duckdb-ORDERED_CTE: [] +POINT_INTERVALS-self-duckdb-OUTER_COUNT: +- - 0 +POINT_INTERVALS-self-duckdb-OUTER_DISTINCT: [] +POINT_INTERVALS-self-duckdb-OUTER_GROUP_BY: [] +POINT_INTERVALS-self-duckdb-OUTER_HAVING: [] +POINT_INTERVALS-self-duckdb-OUTER_LEFT_JOIN: [] +POINT_INTERVALS-self-duckdb-OUTER_ORDER_LIMIT: [] +POINT_INTERVALS-self-duckdb-OUTER_SPATIAL_WHERE: [] +POINT_INTERVALS-self-duckdb-OUTER_WHERE: [] +POINT_INTERVALS-self-duckdb-SELF_JOIN_OVERLAP: [] +POINT_INTERVALS-self-duckdb-SET_DIFFERENCE: [] +POINT_INTERVALS-self-duckdb-SET_UNION: +- - chr1 +- - chr2 +POINT_INTERVALS-self-duckdb-STANDALONE: [] +POINT_INTERVALS-self-duckdb-SUBQUERY_REFERENCE: [] +POINT_INTERVALS-self-duckdb-WINDOWED_PROJECTION: [] +POINT_INTERVALS-self-duckdb-WRAPPING_CTE: [] +REFERENCE_GAP-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 80 +REFERENCE_GAP-reference-duckdb-ALIASED_PROJECTION: +- - 0 + - 40 +- - 60 + - 100 +REFERENCE_GAP-reference-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 60 +REFERENCE_GAP-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - t + - chr1 + - 60 + - 100 +REFERENCE_GAP-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - t + - chr1 + - 60 + - 100 +REFERENCE_GAP-reference-duckdb-JOINED: +- - chr1 + - 0 + - 40 + - g1 +REFERENCE_GAP-reference-duckdb-NESTED_AGGREGATE: +- - 2 + - 80 +REFERENCE_GAP-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - t + - chr1 + - 60 + - 100 +REFERENCE_GAP-reference-duckdb-OUTER_COUNT: +- - 2 +REFERENCE_GAP-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 40 +- - chr1 + - 60 + - 100 +REFERENCE_GAP-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 2 +REFERENCE_GAP-reference-duckdb-OUTER_HAVING: +- - chr1 + - 2 +REFERENCE_GAP-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 60 + - 100 + - null +REFERENCE_GAP-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +REFERENCE_GAP-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +REFERENCE_GAP-reference-duckdb-OUTER_WHERE: [] +REFERENCE_GAP-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 60 + - 100 + - 0 +REFERENCE_GAP-reference-duckdb-SET_DIFFERENCE: [] +REFERENCE_GAP-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr2 +REFERENCE_GAP-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - t + - chr1 + - 60 + - 100 +REFERENCE_GAP-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - t + - chr1 + - 60 + - 100 +REFERENCE_GAP-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 + - 2 +- - chr1 + - 0 + - 100 + - t + - chr1 + - 60 + - 100 + - 2 +REFERENCE_GAP-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - t + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - t + - chr1 + - 60 + - 100 +REFERENCE_GAP-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 180 +REFERENCE_GAP-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 40 +- - 0 + - 40 +- - 40 + - 60 +- - 60 + - 100 +- - 60 + - 100 +REFERENCE_GAP-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 0 +- - 0 +- - 0 +- - 40 +- - 60 +- - 60 +- - 60 +- - 60 +REFERENCE_GAP-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +- - chr1 + - 60 + - 100 + - b + - chr1 + - 60 + - 100 +REFERENCE_GAP-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +- - chr1 + - 60 + - 100 + - b + - chr1 + - 60 + - 100 +REFERENCE_GAP-self-duckdb-JOINED: +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 0 + - 40 + - g1 +REFERENCE_GAP-self-duckdb-NESTED_AGGREGATE: +- - 5 + - 180 +REFERENCE_GAP-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +- - chr1 + - 60 + - 100 + - b + - chr1 + - 60 + - 100 +REFERENCE_GAP-self-duckdb-OUTER_COUNT: +- - 5 +REFERENCE_GAP-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 40 +- - chr1 + - 40 + - 60 +- - chr1 + - 60 + - 100 +REFERENCE_GAP-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 5 +REFERENCE_GAP-self-duckdb-OUTER_HAVING: +- - chr1 + - 5 +REFERENCE_GAP-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 0 + - 40 + - g1 +- - chr1 + - 40 + - 60 + - null +- - chr1 + - 60 + - 100 + - null +- - chr1 + - 60 + - 100 + - null +REFERENCE_GAP-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +REFERENCE_GAP-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +REFERENCE_GAP-self-duckdb-OUTER_WHERE: [] +REFERENCE_GAP-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 0 + - 40 + - 0 +- - chr1 + - 40 + - 60 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 0 +- - chr1 + - 60 + - 100 + - 60 +- - chr1 + - 60 + - 100 + - 60 +REFERENCE_GAP-self-duckdb-SET_DIFFERENCE: [] +REFERENCE_GAP-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr2 +REFERENCE_GAP-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +- - chr1 + - 60 + - 100 + - b + - chr1 + - 60 + - 100 +REFERENCE_GAP-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +- - chr1 + - 60 + - 100 + - b + - chr1 + - 60 + - 100 +REFERENCE_GAP-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 + - 5 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 + - 5 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 60 + - 100 + - 5 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 + - 5 +- - chr1 + - 60 + - 100 + - b + - chr1 + - 60 + - 100 + - 5 +REFERENCE_GAP-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 0 + - 40 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 40 + - 60 +- - chr1 + - 0 + - 100 + - spanning + - chr1 + - 60 + - 100 +- - chr1 + - 0 + - 40 + - a + - chr1 + - 0 + - 40 +- - chr1 + - 60 + - 100 + - b + - chr1 + - 60 + - 100 +REFERENCE_SUPERSET-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 20 +REFERENCE_SUPERSET-reference-duckdb-ALIASED_PROJECTION: +- - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-CHAINED_COOCCURRING: +- - 40 +REFERENCE_SUPERSET-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-JOINED: +- - chr1 + - 40 + - 60 + - g1 +REFERENCE_SUPERSET-reference-duckdb-NESTED_AGGREGATE: +- - 1 + - 20 +REFERENCE_SUPERSET-reference-duckdb-ORDERED_CTE: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-OUTER_COUNT: +- - 1 +REFERENCE_SUPERSET-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 1 +REFERENCE_SUPERSET-reference-duckdb-OUTER_HAVING: [] +REFERENCE_SUPERSET-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 40 + - 60 + - g1 +REFERENCE_SUPERSET-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-OUTER_WHERE: [] +REFERENCE_SUPERSET-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 40 + - 60 + - 40 +REFERENCE_SUPERSET-reference-duckdb-SET_DIFFERENCE: [] +REFERENCE_SUPERSET-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +REFERENCE_SUPERSET-reference-duckdb-STANDALONE: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 + - 1 +REFERENCE_SUPERSET-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 40 + - 60 + - t + - chr1 + - 40 + - 60 +REFERENCE_SUPERSET-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 260 +REFERENCE_SUPERSET-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 20 +- - 20 + - 80 +- - 20 + - 80 +- - 80 + - 200 +REFERENCE_SUPERSET-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 20 +- - 20 +- - 20 +- - 20 +- - 80 +REFERENCE_SUPERSET-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +REFERENCE_SUPERSET-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +REFERENCE_SUPERSET-self-duckdb-JOINED: +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 20 + - 80 + - g1 +- - chr1 + - 20 + - 80 + - g1 +- - chr1 + - 80 + - 200 + - g1 +REFERENCE_SUPERSET-self-duckdb-NESTED_AGGREGATE: +- - 4 + - 260 +REFERENCE_SUPERSET-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +REFERENCE_SUPERSET-self-duckdb-OUTER_COUNT: +- - 4 +REFERENCE_SUPERSET-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 20 +- - chr1 + - 20 + - 80 +- - chr1 + - 80 + - 200 +REFERENCE_SUPERSET-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 4 +REFERENCE_SUPERSET-self-duckdb-OUTER_HAVING: +- - chr1 + - 4 +REFERENCE_SUPERSET-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 20 + - 80 + - g1 +- - chr1 + - 20 + - 80 + - g1 +- - chr1 + - 80 + - 200 + - g1 +REFERENCE_SUPERSET-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 +REFERENCE_SUPERSET-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +REFERENCE_SUPERSET-self-duckdb-OUTER_WHERE: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +REFERENCE_SUPERSET-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 20 + - 0 +- - chr1 + - 20 + - 80 + - 0 +- - chr1 + - 20 + - 80 + - 0 +- - chr1 + - 20 + - 80 + - 20 +- - chr1 + - 20 + - 80 + - 20 +- - chr1 + - 80 + - 200 + - 0 +REFERENCE_SUPERSET-self-duckdb-SET_DIFFERENCE: [] +REFERENCE_SUPERSET-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +REFERENCE_SUPERSET-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +REFERENCE_SUPERSET-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +REFERENCE_SUPERSET-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 + - 4 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 + - 4 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 + - 4 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 + - 4 +REFERENCE_SUPERSET-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 20 + - 80 +- - chr1 + - 0 + - 200 + - wide + - chr1 + - 80 + - 200 +- - chr1 + - 20 + - 80 + - a + - chr1 + - 20 + - 80 +SINGLE_CHROMOSOME-reference-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 80 +SINGLE_CHROMOSOME-reference-duckdb-ALIASED_PROJECTION: +- - 20 + - 60 +- - 60 + - 100 +SINGLE_CHROMOSOME-reference-duckdb-CHAINED_COOCCURRING: +- - 20 +- - 60 +SINGLE_CHROMOSOME-reference-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +- - chr1 + - 0 + - 120 + - t + - chr1 + - 60 + - 100 +SINGLE_CHROMOSOME-reference-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +- - chr1 + - 0 + - 120 + - t + - chr1 + - 60 + - 100 +SINGLE_CHROMOSOME-reference-duckdb-JOINED: +- - chr1 + - 20 + - 60 + - g1 +- - chr1 + - 60 + - 100 + - g1 +SINGLE_CHROMOSOME-reference-duckdb-NESTED_AGGREGATE: +- - 2 + - 80 +SINGLE_CHROMOSOME-reference-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +- - chr1 + - 0 + - 120 + - t + - chr1 + - 60 + - 100 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_COUNT: +- - 2 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_DISTINCT: +- - chr1 + - 20 + - 60 +- - chr1 + - 60 + - 100 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_GROUP_BY: +- - chr1 + - 2 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_HAVING: +- - chr1 + - 2 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 20 + - 60 + - g1 +- - chr1 + - 60 + - 100 + - g1 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +SINGLE_CHROMOSOME-reference-duckdb-OUTER_WHERE: [] +SINGLE_CHROMOSOME-reference-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 20 + - 60 + - 0 +- - chr1 + - 60 + - 100 + - 0 +SINGLE_CHROMOSOME-reference-duckdb-SET_DIFFERENCE: [] +SINGLE_CHROMOSOME-reference-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +SINGLE_CHROMOSOME-reference-duckdb-STANDALONE: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +- - chr1 + - 0 + - 120 + - t + - chr1 + - 60 + - 100 +SINGLE_CHROMOSOME-reference-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +- - chr1 + - 0 + - 120 + - t + - chr1 + - 60 + - 100 +SINGLE_CHROMOSOME-reference-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 + - 2 +- - chr1 + - 0 + - 120 + - t + - chr1 + - 60 + - 100 + - 2 +SINGLE_CHROMOSOME-reference-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 120 + - t + - chr1 + - 20 + - 60 +- - chr1 + - 0 + - 120 + - t + - chr1 + - 60 + - 100 +SINGLE_CHROMOSOME-self-duckdb-AGGREGATED_INTERVAL: +- - chr1 + - 240 +SINGLE_CHROMOSOME-self-duckdb-ALIASED_PROJECTION: +- - 0 + - 20 +- - 100 + - 120 +- - 20 + - 40 +- - 20 + - 40 +- - 40 + - 80 +- - 40 + - 80 +- - 40 + - 80 +- - 80 + - 100 +- - 80 + - 100 +SINGLE_CHROMOSOME-self-duckdb-CHAINED_COOCCURRING: +- - 0 +- - 100 +- - 20 +- - 20 +- - 20 +- - 20 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 40 +- - 80 +- - 80 +- - 80 +- - 80 +SINGLE_CHROMOSOME-self-duckdb-CHAINED_REFERENCE: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 80 + - 100 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 100 + - 120 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 80 + - 100 +SINGLE_CHROMOSOME-self-duckdb-DERIVED_SUBQUERY: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 80 + - 100 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 100 + - 120 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 80 + - 100 +SINGLE_CHROMOSOME-self-duckdb-JOINED: +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 100 + - 120 + - g1 +- - chr1 + - 20 + - 40 + - g1 +- - chr1 + - 20 + - 40 + - g1 +- - chr1 + - 40 + - 80 + - g1 +- - chr1 + - 40 + - 80 + - g1 +- - chr1 + - 40 + - 80 + - g1 +- - chr1 + - 80 + - 100 + - g1 +- - chr1 + - 80 + - 100 + - g1 +SINGLE_CHROMOSOME-self-duckdb-NESTED_AGGREGATE: +- - 9 + - 240 +SINGLE_CHROMOSOME-self-duckdb-ORDERED_CTE: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 80 + - 100 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 100 + - 120 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 80 + - 100 +SINGLE_CHROMOSOME-self-duckdb-OUTER_COUNT: +- - 9 +SINGLE_CHROMOSOME-self-duckdb-OUTER_DISTINCT: +- - chr1 + - 0 + - 20 +- - chr1 + - 100 + - 120 +- - chr1 + - 20 + - 40 +- - chr1 + - 40 + - 80 +- - chr1 + - 80 + - 100 +SINGLE_CHROMOSOME-self-duckdb-OUTER_GROUP_BY: +- - chr1 + - 9 +SINGLE_CHROMOSOME-self-duckdb-OUTER_HAVING: +- - chr1 + - 9 +SINGLE_CHROMOSOME-self-duckdb-OUTER_LEFT_JOIN: +- - chr1 + - 0 + - 20 + - g1 +- - chr1 + - 100 + - 120 + - g1 +- - chr1 + - 20 + - 40 + - g1 +- - chr1 + - 20 + - 40 + - g1 +- - chr1 + - 40 + - 80 + - g1 +- - chr1 + - 40 + - 80 + - g1 +- - chr1 + - 40 + - 80 + - g1 +- - chr1 + - 80 + - 100 + - g1 +- - chr1 + - 80 + - 100 + - g1 +SINGLE_CHROMOSOME-self-duckdb-OUTER_ORDER_LIMIT: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 +SINGLE_CHROMOSOME-self-duckdb-OUTER_SPATIAL_WHERE: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 +SINGLE_CHROMOSOME-self-duckdb-OUTER_WHERE: [] +SINGLE_CHROMOSOME-self-duckdb-SELF_JOIN_OVERLAP: +- - chr1 + - 0 + - 20 + - 0 +- - chr1 + - 100 + - 120 + - 40 +- - chr1 + - 20 + - 40 + - 0 +- - chr1 + - 20 + - 40 + - 0 +- - chr1 + - 20 + - 40 + - 20 +- - chr1 + - 20 + - 40 + - 20 +- - chr1 + - 40 + - 80 + - 0 +- - chr1 + - 40 + - 80 + - 0 +- - chr1 + - 40 + - 80 + - 0 +- - chr1 + - 40 + - 80 + - 20 +- - chr1 + - 40 + - 80 + - 20 +- - chr1 + - 40 + - 80 + - 20 +- - chr1 + - 40 + - 80 + - 40 +- - chr1 + - 40 + - 80 + - 40 +- - chr1 + - 40 + - 80 + - 40 +- - chr1 + - 80 + - 100 + - 20 +- - chr1 + - 80 + - 100 + - 20 +- - chr1 + - 80 + - 100 + - 40 +- - chr1 + - 80 + - 100 + - 40 +SINGLE_CHROMOSOME-self-duckdb-SET_DIFFERENCE: [] +SINGLE_CHROMOSOME-self-duckdb-SET_UNION: +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +- - chr1 +SINGLE_CHROMOSOME-self-duckdb-STANDALONE: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 80 + - 100 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 100 + - 120 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 80 + - 100 +SINGLE_CHROMOSOME-self-duckdb-SUBQUERY_REFERENCE: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 80 + - 100 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 100 + - 120 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 80 + - 100 +SINGLE_CHROMOSOME-self-duckdb-WINDOWED_PROJECTION: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 + - 9 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 + - 9 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 + - 9 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 + - 9 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 + - 9 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 80 + - 100 + - 9 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 100 + - 120 + - 9 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 + - 9 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 80 + - 100 + - 9 +SINGLE_CHROMOSOME-self-duckdb-WRAPPING_CTE: +- - chr1 + - 0 + - 80 + - a + - chr1 + - 0 + - 20 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 20 + - 40 +- - chr1 + - 0 + - 80 + - a + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 20 + - 40 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 40 + - 80 +- - chr1 + - 20 + - 100 + - b + - chr1 + - 80 + - 100 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 100 + - 120 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 40 + - 80 +- - chr1 + - 40 + - 120 + - c + - chr1 + - 80 + - 100 diff --git a/tests/test_usage_patterns.py b/tests/test_usage_patterns.py new file mode 100644 index 0000000..545aad7 --- /dev/null +++ b/tests/test_usage_patterns.py @@ -0,0 +1,380 @@ +"""Functional suite for the GIQL operator usage-pattern catalogue. + +Executes every catalogued DISJOIN usage pattern's transpiled SQL against a +matrix of database engines and snapshots the result rows with pytest-manifest, +verifying each pattern is valid SQL and functionally correct in its query +context. The functional matrix crosses two dimensions: the usage patterns of +``usage_patterns.py`` and the :data:`~tests.usage_patterns.DISJOIN_PROFILES` +semantic profiles -- one per DISJOIN edge case. The committed snapshots under +``tests/manifests/`` are the oracle: regenerate with ``pytest -O`` and review +the diff before committing. + +Patterns GIQL cannot transpile (a subquery, CTE, or nested operator result in a +table-function's target position) are catalogued but strict-``xfail``. + +The module also carries unit-tier guard tests for the catalogue's descriptor +machinery: :class:`TestTableFixture`, :class:`TestRenderTableFunction`, and +:class:`TestDisjoinProfiles`. +""" + +import pytest + +from giql import transpile +from giql.table import Table + +from .usage_patterns import CANONICAL_COLUMNS +from .usage_patterns import DISJOIN_PROFILES +from .usage_patterns import DISJOIN_USAGE +from .usage_patterns import OperatorType +from .usage_patterns import OperatorUsage +from .usage_patterns import TableFixture +from .usage_patterns import UsagePattern +from .usage_patterns import render +from .usage_patterns import templated_patterns + + +class _TemplatelessUsage(OperatorUsage): + """An OperatorUsage subclass with an empty template table, for guard tests.""" + + name = "NO_OP" + operator_type = OperatorType.TABLE_FUNCTION + + @property + def tables(self): + """Return no tables -- this descriptor is never executed.""" + return () + + def template_table(self): + """Return an empty template table so render() always rejects.""" + return {} + + def slots(self): + """Return no slot values -- no template is ever rendered.""" + return {} + + +def _run_duckdb(fixtures: tuple[TableFixture, ...], sql: str) -> list: + """Load `fixtures` into in-memory DuckDB, execute `sql`, return the rows. + + Each fixture's physical schema (`fixture.columns`) drives the `CREATE + TABLE`, so a fixture carrying a custom `Table` config is materialized with + its own physical column names and types. + """ + import duckdb + + conn = duckdb.connect(":memory:") + try: + for fixture in fixtures: + column_ddl = ", ".join( + f'"{column}" {sql_type}' for column, sql_type in fixture.columns + ) + conn.execute(f'CREATE TABLE "{fixture.name}" ({column_ddl})') + if fixture.rows: + placeholders = ", ".join("?" for _ in fixture.columns) + conn.executemany( + f'INSERT INTO "{fixture.name}" VALUES ({placeholders})', + [list(row) for row in fixture.rows], + ) + return conn.execute(sql).fetchall() + finally: + conn.close() + + +# Database engines the functional matrix runs against. DuckDB only for now; +# SQLite and PostgreSQL are future entries keyed in here. +ENGINES = {"duckdb": _run_duckdb} + + +def _execute(usage: OperatorUsage, query: str, engine: str) -> list: + """Transpile `query`, run it on `engine`, return rows sorted for stability. + + The descriptor's `tables` yields a `Table` config for any fixture with a + custom physical layout, so GIQL resolves custom column names and + coordinate systems during transpilation. + """ + sql = transpile(query, tables=list(usage.tables)) + rows = ENGINES[engine](usage.fixtures, sql) + return sorted(([list(row) for row in rows]), key=repr) + + +def _profile_cases() -> list: + """Return (profile, mode, usage, pattern, query, engine) params for the matrix. + + One param is produced per (profile, mode, pattern, engine). Each param + carries a stable id, the pattern's `usage` mark, and -- for an + untranspilable pattern -- a strict `xfail`. Reference-mode and self-mode + are flattened into one parametrization so a single test method covers the + whole matrix; `mode` is threaded into the manifest key so a profile's two + modes never collide on the same snapshot row. + """ + cases: list = [] + for profile in DISJOIN_PROFILES: + modes = (("self", profile.self_usage), ("reference", profile.reference_usage)) + for mode, usage in modes: + for pattern in templated_patterns(usage): + marks = [pytest.mark.usage(pattern), pytest.mark.integration] + if pattern in usage.xfails: + marks.append( + pytest.mark.xfail( + reason=usage.xfails[pattern], strict=True + ) + ) + for engine in ENGINES: + cases.append( + pytest.param( + profile.profile_id, + mode, + usage, + pattern, + render(pattern, usage), + engine, + id=( + f"{profile.profile_id}-{mode}-" + f"{engine}-{pattern.name}" + ), + marks=marks, + ) + ) + return cases + + +class TestDisjoinUsageMatrix: + """Every DISJOIN usage pattern executes to a snapshotted result per profile.""" + + @pytest.mark.parametrize( + ("profile_id", "mode", "usage", "pattern", "query", "engine"), + _profile_cases(), + ) + def test_disjoin_pattern_executes_to_snapshot_across_profiles( + self, profile_id, mode, usage, pattern, query, engine, manifest + ): + """Test every DISJOIN usage pattern executes to its per-profile snapshot. + + Given: + A canonical usage pattern rendered for one DISJOIN semantic profile + in either self-mode or reference-mode. + When: + Its transpiled SQL is executed against the engine. + Then: + It should produce result rows matching the captured manifest + snapshot keyed by profile, mode, engine, and pattern. + """ + # Arrange + key = f"{profile_id}-{mode}-{engine}-{pattern.name}" + + # Act + rows = _execute(usage, query, engine) + + # Assert + assert manifest[key] == rows + + def test_templated_patterns_should_cover_every_canonical_pattern(self): + """Test the DISJOIN catalogue templates every canonical pattern. + + Given: + The canonical table-function usage-pattern list, including the + OUTER_COUNT, OUTER_LEFT_JOIN, SELF_JOIN_OVERLAP, NESTED_AGGREGATE, + and ORDERED_CTE members. + When: + Collecting the patterns the DISJOIN catalogue can render. + Then: + It should render every canonical pattern, the new members + included. + """ + # Act + renderable = set(templated_patterns(DISJOIN_USAGE)) + + # Assert + assert renderable == set(DISJOIN_USAGE.canonical_patterns) + new_members = { + UsagePattern.OUTER_COUNT, + UsagePattern.OUTER_LEFT_JOIN, + UsagePattern.SELF_JOIN_OVERLAP, + UsagePattern.NESTED_AGGREGATE, + UsagePattern.ORDERED_CTE, + } + assert new_members <= renderable + + +class TestTableFixture: + """Tests for the TableFixture catalogue fixture descriptor.""" + + def test___init___with_name_and_rows_only(self): + """Test a TableFixture defaults to the canonical schema and no Table. + + Given: + A TableFixture constructed with only a name and rows. + When: + The fixture is inspected. + Then: + It should expose the canonical column schema and a None Table + config. + """ + # Act + fixture = TableFixture("peaks", (("chr1", 0, 10, "p"),)) + + # Assert + assert fixture.columns == CANONICAL_COLUMNS + assert fixture.table is None + + def test___init___with_custom_table_and_schema(self): + """Test a TableFixture exposes a supplied Table config and column schema. + + Given: + A TableFixture constructed with a custom Table config and an + explicit physical column schema. + When: + The fixture is inspected. + Then: + It should expose both the custom schema and the Table config. + """ + # Arrange + schema = (("seqid", "VARCHAR"), ("lo", "INTEGER"), ("hi", "INTEGER")) + config = Table("v", chrom_col="seqid", start_col="lo", end_col="hi") + + # Act + fixture = TableFixture("v", (("chr1", 1, 9),), columns=schema, table=config) + + # Assert + assert fixture.columns == schema + assert fixture.table is config + + def test___init___with_zero_rows(self): + """Test a TableFixture with no rows is a valid empty fixture. + + Given: + A TableFixture constructed with an empty rows tuple. + When: + The fixture is inspected. + Then: + It should be a valid fixture exposing zero rows and the canonical + schema. + """ + # Act + fixture = TableFixture("empty", ()) + + # Assert + assert fixture.rows == () + assert fixture.columns == CANONICAL_COLUMNS + + +class TestRenderTableFunction: + """Tests for render() applied to table-function usage descriptors.""" + + def test_render_with_new_member_pattern(self): + """Test render() produces a complete query for a new-member pattern. + + Given: + A DISJOIN_PROFILES descriptor and the NESTED_AGGREGATE usage + pattern (a member added for issue #87). + When: + The pattern is rendered through render(). + Then: + It should return a complete query string with every slot filled + and no unsubstituted braces. + """ + # Act + query = render(UsagePattern.NESTED_AGGREGATE, DISJOIN_USAGE) + + # Assert + assert "{" not in query and "}" not in query + assert "DISJOIN(features)" in query + assert "disjoin_end" in query and "disjoin_start" in query + + def test_render_with_pattern_having_no_template(self): + """Test render() rejects a pattern its operator class cannot template. + + Given: + An operator usage descriptor whose class has an empty template + table, and a usage pattern absent from it. + When: + render() is called for that pattern. + Then: + It should raise ValueError naming the operator and the pattern. + """ + # Arrange + templateless = _TemplatelessUsage() + + # Act & assert + with pytest.raises(ValueError, match=r"NO_OP.*NESTED_AGGREGATE"): + render(UsagePattern.NESTED_AGGREGATE, templateless) + + +class TestDisjoinProfiles: + """Tests for the DISJOIN_PROFILES semantic-profile collection.""" + + @pytest.mark.parametrize("profile", DISJOIN_PROFILES, ids=lambda p: p.profile_id) + def test_profile_patterns_reference_only_registered_fixtures(self, profile): + """Test every table a profile's patterns reference is a known fixture. + + Given: + A DISJOIN profile and its self-mode and reference-mode usages. + When: + Collecting every table name the rendered patterns reference. + Then: + It should reference only tables present in the usage's fixtures. + """ + # Arrange + for usage in (profile.self_usage, profile.reference_usage): + fixture_names = {fixture.name for fixture in usage.fixtures} + referenced = { + usage.target, + usage.join_table, + usage.ref_operand, + } + + # Act & assert + assert referenced <= fixture_names + + @pytest.mark.parametrize("profile", DISJOIN_PROFILES, ids=lambda p: p.profile_id) + def test_profile_names_target_and_reference_in_fixtures(self, profile): + """Test every profile names a target and reference present in fixtures. + + Given: + A DISJOIN profile's self-mode and reference-mode usages. + When: + Resolving the declared target and reference table names. + Then: + It should find the target in self-mode fixtures and both the + target and the reference in reference-mode fixtures. + """ + # Arrange + self_names = {f.name for f in profile.self_usage.fixtures} + ref_names = {f.name for f in profile.reference_usage.fixtures} + + # Act & assert + assert profile.self_usage.target in self_names + assert profile.reference_usage.target in ref_names + assert profile.reference_usage.reference in ref_names + + def test_profile_ids_are_unique(self): + """Test every DISJOIN profile id is unique across the collection. + + Given: + The DISJOIN_PROFILES collection, whose ids prefix manifest keys. + When: + Collecting every profile id. + Then: + It should contain no duplicate id, so manifest keys never collide. + """ + # Act + ids = [profile.profile_id for profile in DISJOIN_PROFILES] + + # Assert + assert len(ids) == len(set(ids)) + + def test_profile_ids_match_nested_usage_profile_ids(self): + """Test each profile's id matches the profile_id of its bound usages. + + Given: + A DISJOIN profile and the two TableFunctionUsage descriptors it + binds. + When: + Comparing the profile id to each usage's profile_id field. + Then: + It should match on both the self-mode and reference-mode usage. + """ + # Act & assert + for profile in DISJOIN_PROFILES: + assert profile.self_usage.profile_id == profile.profile_id + assert profile.reference_usage.profile_id == profile.profile_id diff --git a/tests/usage_patterns.py b/tests/usage_patterns.py new file mode 100644 index 0000000..2b7f8c5 --- /dev/null +++ b/tests/usage_patterns.py @@ -0,0 +1,1051 @@ +"""Canonical query-usage patterns for GIQL operators. + +A shared catalogue of the ways an operator can appear *within* a SQL query -- +standalone, filtered, joined, wrapped in a CTE, and so on. The functional suite +(``test_usage_patterns.py``) parametrises over this catalogue, executing every +pattern against a matrix of database engines and snapshotting the results. + +Each operator class has a descriptor -- an :class:`OperatorUsage` subclass -- +carrying its query templates, its ``str.format`` slot values, the fixture data +its queries run against, and its canonical pattern list. :func:`render`, +:func:`templated_patterns`, and :func:`rendered_cases` dispatch through the +descriptor, so a new operator class is a new subclass with no edits to those +helpers. + +Only the table-function class (``DISJOIN``) is populated. ``AggregateUsage``, +``PredicateUsage``, and ``ScalarUsage`` are defined but unpopulated extension +points. + +The DISJOIN catalogue is crossed against a second dimension: the +:data:`DISJOIN_PROFILES` collection of named semantic profiles, one per +DISJOIN edge case (point intervals, adjacent intervals, custom column names, +non-canonical encodings, and so on). The functional suite parametrises every +pattern over every profile, snapshotting one manifest per profile. + +:func:`rendered_cases` tags each case with ``@pytest.mark.usage`` and, for a +pattern GIQL cannot transpile, a strict ``xfail``. +""" + +from __future__ import annotations + +import abc +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum +from types import MappingProxyType +from typing import ClassVar + +import pytest + +from giql.table import Table + + +class OperatorType(Enum): + """The structural class of a GIQL operator -- the query position it occupies.""" + + TABLE_FUNCTION = "table_function" + """A FROM-clause table function yielding rows (DISJOIN, NEAREST).""" + + AGGREGATE = "aggregate" + """A SELECT-list aggregate or window operator (MERGE, CLUSTER).""" + + PREDICATE = "predicate" + """A boolean predicate in WHERE / ON / HAVING (INTERSECTS, CONTAINS, WITHIN).""" + + SCALAR = "scalar" + """A scalar function usable in any value expression (DISTANCE).""" + + +class UsagePattern(Enum): + """A canonical way of placing or combining an operator within a query.""" + + STANDALONE = "standalone" + """The operator is the sole FROM source: ``SELECT * FROM OP(...)``.""" + + ALIASED_PROJECTION = "aliased_projection" + """The result is aliased and specific output columns are projected.""" + + OUTER_WHERE = "outer_where" + """An outer ``WHERE`` filters the result on a real predicate.""" + + OUTER_ORDER_LIMIT = "outer_order_limit" + """An outer ``ORDER BY`` / ``LIMIT`` is applied to the result.""" + + OUTER_GROUP_BY = "outer_group_by" + """An outer ``GROUP BY`` aggregates the result by an output column.""" + + OUTER_HAVING = "outer_having" + """An outer ``GROUP BY`` ... ``HAVING`` filters the grouped result.""" + + OUTER_COUNT = "outer_count" + """An outer ``SELECT COUNT(*)`` collapses the result to a single row.""" + + AGGREGATED_INTERVAL = "aggregated_interval" + """An aggregate over the operator's derived interval columns (coverage).""" + + OUTER_DISTINCT = "outer_distinct" + """``SELECT DISTINCT`` is applied over the result.""" + + OUTER_SPATIAL_WHERE = "outer_spatial_where" + """An outer interval-overlap predicate filters the result to a region.""" + + WINDOWED_PROJECTION = "windowed_projection" + """A window function is computed over the operator's result.""" + + WRAPPING_CTE = "wrapping_cte" + """The operator is wrapped in a CTE and consumed downstream.""" + + ORDERED_CTE = "ordered_cte" + """The operator is wrapped in a CTE with an inner ``ORDER BY``, consumed outside.""" + + DERIVED_SUBQUERY = "derived_subquery" + """The operator's result is nested as a derived-table subquery.""" + + NESTED_AGGREGATE = "nested_aggregate" + """An aggregate consumes a derived-table subquery of the result.""" + + JOINED = "joined" + """The result is interval-overlap-joined to another relation.""" + + OUTER_LEFT_JOIN = "outer_left_join" + """The result is ``LEFT JOIN``-ed to another relation.""" + + SELF_JOIN_OVERLAP = "self_join_overlap" + """The result is interval-overlap-joined back to the operator's base table.""" + + SET_UNION = "set_union" + """The result is combined with another relation via ``UNION``.""" + + SET_DIFFERENCE = "set_difference" + """The result is combined with another relation via ``EXCEPT``.""" + + SUBQUERY_INPUT = "subquery_input" + """The operator's target is a derived-table subquery.""" + + CTE_INPUT = "cte_input" + """The operator's target is a CTE defined in the outer query.""" + + SUBQUERY_REFERENCE = "subquery_reference" + """The operator's ``reference`` argument is a subquery.""" + + CHAINED_INPUT = "chained_input" + """The operator's target is another operator's result.""" + + CHAINED_REFERENCE = "chained_reference" + """The operator's ``reference`` argument is another operator's result.""" + + CHAINED_COOCCURRING = "chained_cooccurring" + """Two GIQL operator calls co-occur in one query.""" + + +# Reasons GIQL cannot transpile certain patterns -- attached as xfail reasons so +# the catalogue enumerates the expected use while documenting the limitation. +_TARGET_MUST_BE_TABLE = ( + "GIQL requires a table-function target to be a registered base table; " + "a subquery or CTE in target position is rejected during target resolution" +) +_NESTED_OPERATOR_TARGET = ( + "GIQL does not recognise a nested operator call in a table-function's " + "target position; the inner call is emitted as a mangled identifier" +) + +# SQL templates for the table-function class. Brace slots are filled from a +# TableFunctionUsage by render(). Aliases use a __ prefix to avoid colliding +# with operator-generated identifiers. +_TABLE_FUNCTION_TEMPLATES: dict[UsagePattern, str] = { + UsagePattern.STANDALONE: "SELECT * FROM {op}", + UsagePattern.ALIASED_PROJECTION: ( + "SELECT __d.{out_start} AS s, __d.{out_end} AS e FROM {op} AS __d" + ), + UsagePattern.OUTER_WHERE: ( + "SELECT * FROM {op} WHERE {out_end} - {out_start} >= 50" + ), + UsagePattern.OUTER_ORDER_LIMIT: ( + "SELECT * FROM {op} ORDER BY {out_start} LIMIT 1" + ), + UsagePattern.OUTER_GROUP_BY: ( + "SELECT {out_chrom}, COUNT(*) AS n FROM {op} GROUP BY {out_chrom}" + ), + UsagePattern.OUTER_HAVING: ( + "SELECT {out_chrom}, COUNT(*) AS n FROM {op} " + "GROUP BY {out_chrom} HAVING COUNT(*) > 1" + ), + # A scalar COUNT(*) over the whole result: collapses to one deterministic + # row regardless of the result's own row order. + UsagePattern.OUTER_COUNT: "SELECT COUNT(*) AS n FROM {op}", + UsagePattern.AGGREGATED_INTERVAL: ( + "SELECT {out_chrom}, SUM({out_end} - {out_start}) AS total_bp " + "FROM {op} GROUP BY {out_chrom}" + ), + UsagePattern.OUTER_DISTINCT: ( + "SELECT DISTINCT {out_chrom}, {out_start}, {out_end} FROM {op}" + ), + # A region filter over the operator's output. GIQL's INTERSECTS does not + # work here: applied to a table-function alias it silently resolves to the + # alias's default chrom/start/end (the passed-through parent interval), not + # the disjoin_* segment -- so an explicit half-open overlap predicate is + # the correct spatial filter on the operator's own output. + UsagePattern.OUTER_SPATIAL_WHERE: ( + "SELECT * FROM {op} " + "WHERE {out_chrom} = 'chr1' AND {out_start} < 50 AND {out_end} > 20" + ), + # COUNT(*) OVER, not ROW_NUMBER(): a window aggregate is deterministic, + # whereas ROW_NUMBER over a non-total ORDER BY (duplicate segments share a + # start) would assign ranks non-deterministically and flake the snapshot. + UsagePattern.WINDOWED_PROJECTION: ( + "SELECT *, COUNT(*) OVER (PARTITION BY {out_chrom}) AS chrom_n " + "FROM {op}" + ), + UsagePattern.WRAPPING_CTE: ( + "WITH __u AS (SELECT * FROM {op}) SELECT * FROM __u" + ), + # An inner ORDER BY inside the CTE. The outer query re-selects unordered; + # the functional suite re-sorts every result by repr, so the snapshot is + # stable even though the inner ORDER BY is not total. + UsagePattern.ORDERED_CTE: ( + "WITH __o AS (SELECT * FROM {op} ORDER BY {out_start}) " + "SELECT * FROM __o" + ), + UsagePattern.DERIVED_SUBQUERY: ( + "SELECT * FROM (SELECT * FROM {op}) AS __s" + ), + # An aggregate consuming a derived-table subquery of the result. COUNT(*) + # and SUM are order-insensitive, so the single output row is deterministic. + UsagePattern.NESTED_AGGREGATE: ( + "SELECT COUNT(*) AS n, SUM({out_end} - {out_start}) AS total_bp " + "FROM (SELECT * FROM {op}) AS __s" + ), + UsagePattern.JOINED: ( + "SELECT __d.{out_chrom}, __d.{out_start}, __d.{out_end}, __o.name " + "FROM {op} AS __d JOIN {join_table} AS __o " + "ON __o.chrom = __d.{out_chrom} " + "AND __o.start < __d.{out_end} AND __o.end > __d.{out_start}" + ), + # A LEFT JOIN: every result sub-interval survives, with NULLs where the + # join_table has no overlapping row. The functional suite re-sorts rows. + UsagePattern.OUTER_LEFT_JOIN: ( + "SELECT __d.{out_chrom}, __d.{out_start}, __d.{out_end}, __o.name " + "FROM {op} AS __d LEFT JOIN {join_table} AS __o " + "ON __o.chrom = __d.{out_chrom} " + "AND __o.start < __d.{out_end} AND __o.end > __d.{out_start}" + ), + # The result interval-overlap-joined back to its own base target table -- + # a real relation join, not a nested operator call. The join addresses the + # target's physical genomic columns (custom for a custom-column target); + # the projected parent start proves a real target row was matched. + UsagePattern.SELF_JOIN_OVERLAP: ( + "SELECT __d.{out_chrom}, __d.{out_start}, __d.{out_end}, " + "__t.{tgt_start} AS parent_start " + "FROM {op} AS __d JOIN {target} AS __t " + "ON __t.{tgt_chrom} = __d.{out_chrom} " + "AND __t.{tgt_start} < __d.{out_end} " + "AND __t.{tgt_end} > __d.{out_start}" + ), + UsagePattern.SET_UNION: ( + "SELECT {out_chrom} AS c FROM {op} " + "UNION ALL SELECT chrom AS c FROM {join_table}" + ), + UsagePattern.SET_DIFFERENCE: ( + "SELECT {out_chrom} AS c FROM {op} " + "EXCEPT SELECT chrom AS c FROM {join_table}" + ), + UsagePattern.SUBQUERY_INPUT: ( + "SELECT * FROM {name}((SELECT * FROM {target}))" + ), + UsagePattern.CTE_INPUT: ( + "WITH __in AS (SELECT * FROM {target}) SELECT * FROM {name}(__in)" + ), + # The reference subquery projects the operand's genomic columns aliased to + # the canonical chrom/start/end names. GIQL's subquery-reference resolution + # assumes a canonical layout, so a custom-column operand must be aliased + # explicitly here rather than passed through with SELECT *. + UsagePattern.SUBQUERY_REFERENCE: ( + "SELECT * FROM {name}({target}, " + "reference := (SELECT {ref_select} FROM {ref_operand}))" + ), + UsagePattern.CHAINED_INPUT: ( + "SELECT * FROM {name}((SELECT * FROM {name}({target})))" + ), + # The reference is another DISJOIN call; its output exposes the + # convention-fixed disjoin_* columns, aliased here to canonical names so + # the outer DISJOIN's subquery-reference resolution can resolve them. + UsagePattern.CHAINED_REFERENCE: ( + "SELECT * FROM {name}({target}, " + "reference := (SELECT {out_chrom} AS chrom, {out_start} AS \"start\", " + "{out_end} AS \"end\" FROM {name}({ref_operand})))" + ), + UsagePattern.CHAINED_COOCCURRING: ( + "SELECT __a.{out_start} FROM {op} AS __a JOIN {op} AS __b " + "ON __a.{out_chrom} = __b.{out_chrom} " + "AND __a.{out_start} < __b.{out_end} AND __a.{out_end} > __b.{out_start}" + ), +} + + +# The canonical physical schema for an interval table: the standard genomic +# layout the original DISJOIN fixtures and the bedtools-compatible defaults +# assume. A TableFixture with no explicit `columns` uses this. +CANONICAL_COLUMNS: tuple[tuple[str, str], ...] = ( + ("chrom", "VARCHAR"), + ("start", "INTEGER"), + ("end", "INTEGER"), + ("name", "VARCHAR"), +) + + +@dataclass(frozen=True) +class TableFixture: + """Fixture rows for one table the functional suite loads before executing. + + ``columns`` is the physical schema as ``(name, sql_type)`` pairs; it + defaults to :data:`CANONICAL_COLUMNS` (``chrom`` VARCHAR, ``start`` + INTEGER, ``end`` INTEGER, ``name`` VARCHAR). Each row tuple must match the + declared schema width and column order. + + ``table`` is an optional :class:`giql.table.Table` configuration. When + present it carries custom column names and/or a non-canonical coordinate + system, and the functional suite passes it to ``transpile`` so GIQL + resolves the fixture's physical layout. When ``None`` the table uses + GIQL's default canonical mapping. + """ + + name: str + rows: tuple[tuple, ...] + columns: tuple[tuple[str, str], ...] = CANONICAL_COLUMNS + table: Table | None = None + + +class OperatorUsage(abc.ABC): + """Descriptor base for one operator's usage-pattern slot values and fixtures. + + One subclass exists per :class:`OperatorType`. A subclass is a frozen + dataclass providing, as attributes/fields, ``name`` (the operator name), + ``operator_type``, ``canonical_patterns`` (the patterns its class is + expected to cover), ``xfails`` (patterns GIQL cannot transpile, mapped to a + reason), and ``fixtures`` (the table data its queries run against). It + implements :meth:`tables`, :meth:`template_table`, and :meth:`slots`. + + :func:`render`, :func:`templated_patterns`, and :func:`rendered_cases` + dispatch through those members, so adding an operator class needs no edits + to the helpers themselves. + """ + + operator_type: ClassVar[OperatorType] + canonical_patterns: ClassVar[tuple[UsagePattern, ...]] = () + xfails: ClassVar[Mapping[UsagePattern, str]] = MappingProxyType({}) + + @property + @abc.abstractmethod + def tables(self) -> tuple[str | Table, ...]: + """Return the table configs to register when transpiling this operator.""" + + @abc.abstractmethod + def template_table(self) -> dict[UsagePattern, str]: + """Return the ``{pattern: SQL template}`` table for this operator's class.""" + + @abc.abstractmethod + def slots(self) -> dict[str, str]: + """Return the ``str.format`` keyword values for this operator's templates.""" + + +@dataclass(frozen=True) +class TableFunctionUsage(OperatorUsage): + """Usage descriptor for a FROM-clause table-function operator (e.g. DISJOIN). + + ``target`` must be a registered base table -- GIQL rejects a subquery or CTE + in target position. ``reference`` is the optional two-table-mode reference + relation; ``join_table`` is a second interval table used by the ``JOINED`` + and set-operation patterns. ``fixtures`` supplies the rows of every table + the patterns reference. + + ``profile_id`` names the semantic edge case the descriptor exercises; it is + the manifest-key prefix that keeps every profile's snapshot rows distinct. + """ + + name: str + target: str + out_chrom: str + out_start: str + out_end: str + fixtures: tuple[TableFixture, ...] + reference: str | None = None + join_table: str = "annotations" + profile_id: str = "CANONICAL" + + operator_type: ClassVar[OperatorType] = OperatorType.TABLE_FUNCTION + canonical_patterns: ClassVar[tuple[UsagePattern, ...]] = ( + UsagePattern.STANDALONE, + UsagePattern.ALIASED_PROJECTION, + UsagePattern.OUTER_WHERE, + UsagePattern.OUTER_ORDER_LIMIT, + UsagePattern.OUTER_GROUP_BY, + UsagePattern.OUTER_HAVING, + UsagePattern.OUTER_COUNT, + UsagePattern.AGGREGATED_INTERVAL, + UsagePattern.OUTER_DISTINCT, + UsagePattern.OUTER_SPATIAL_WHERE, + UsagePattern.WINDOWED_PROJECTION, + UsagePattern.WRAPPING_CTE, + UsagePattern.ORDERED_CTE, + UsagePattern.DERIVED_SUBQUERY, + UsagePattern.NESTED_AGGREGATE, + UsagePattern.JOINED, + UsagePattern.OUTER_LEFT_JOIN, + UsagePattern.SELF_JOIN_OVERLAP, + UsagePattern.SET_UNION, + UsagePattern.SET_DIFFERENCE, + UsagePattern.SUBQUERY_INPUT, + UsagePattern.CTE_INPUT, + UsagePattern.SUBQUERY_REFERENCE, + UsagePattern.CHAINED_INPUT, + UsagePattern.CHAINED_REFERENCE, + UsagePattern.CHAINED_COOCCURRING, + ) + xfails: ClassVar[Mapping[UsagePattern, str]] = MappingProxyType( + { + UsagePattern.SUBQUERY_INPUT: _TARGET_MUST_BE_TABLE, + UsagePattern.CTE_INPUT: _TARGET_MUST_BE_TABLE, + UsagePattern.CHAINED_INPUT: _NESTED_OPERATOR_TARGET, + } + ) + + @property + def op(self) -> str: + """Return the operator call as it appears in a FROM clause.""" + if self.reference is None: + return f"{self.name}({self.target})" + return f"{self.name}({self.target}, reference := {self.reference})" + + @property + def ref_operand(self) -> str: + """Return the relation a reference-bearing template should reference.""" + return self.target if self.reference is None else self.reference + + def _genomic_columns(self, table_name: str) -> tuple[str, str, str]: + """Return the ``(chrom, start, end)`` physical column names of a fixture. + + A fixture carrying a :class:`giql.table.Table` config reports that + config's custom column names; otherwise the canonical ``chrom`` / + ``start`` / ``end`` layout is assumed. Each name is double-quoted so a + template can interpolate it safely even when the physical column is a + SQL reserved word (the canonical layout's ``start`` and ``end`` are). + """ + for fixture in self.fixtures: + if fixture.name == table_name and fixture.table is not None: + config = fixture.table + names = (config.chrom_col, config.start_col, config.end_col) + break + else: + names = ("chrom", "start", "end") + return tuple(f'"{name}"' for name in names) + + @property + def tables(self) -> tuple[str | Table, ...]: + """Return the table configs this operator's patterns reference. + + A fixture carrying a custom :class:`giql.table.Table` config yields the + ``Table`` object so GIQL resolves its physical columns and coordinate + system; a fixture with the canonical layout yields its bare name. + """ + return tuple( + fixture.table if fixture.table is not None else fixture.name + for fixture in self.fixtures + ) + + def template_table(self) -> dict[UsagePattern, str]: + """Return the table-function template table.""" + return _TABLE_FUNCTION_TEMPLATES + + def slots(self) -> dict[str, str]: + """Return the template slot values for this table-function usage. + + ``tgt_chrom`` / ``tgt_start`` / ``tgt_end`` are the target table's + physical genomic column names (custom when the target fixture carries + a :class:`giql.table.Table` config), so a template that joins the + result back to the base target table addresses real columns. + ``ref_select`` is a canonical-column projection of the reference + operand: it aliases the operand's physical columns to ``chrom`` / + ``start`` / ``end``, the column names GIQL's subquery / CTE reference + resolution requires. + """ + tgt_chrom, tgt_start, tgt_end = self._genomic_columns(self.target) + ref_chrom, ref_start, ref_end = self._genomic_columns(self.ref_operand) + ref_select = ( + f'{ref_chrom} AS chrom, {ref_start} AS "start", ' + f'{ref_end} AS "end"' + ) + return { + "name": self.name, + "op": self.op, + "target": self.target, + "ref_operand": self.ref_operand, + "ref_select": ref_select, + "out_chrom": self.out_chrom, + "out_start": self.out_start, + "out_end": self.out_end, + "tgt_chrom": tgt_chrom, + "tgt_start": tgt_start, + "tgt_end": tgt_end, + "join_table": self.join_table, + } + + +class AggregateUsage(OperatorUsage): + """Extension point -- SELECT-list aggregate operators (MERGE, CLUSTER). + + Not yet populated. To populate: make this a frozen dataclass with slots for + the aggregate call, the source table, an optional result alias (the window + form), and operator arguments; set ``name``, ``canonical_patterns`` (an + aggregate-specific pattern set), and ``fixtures``; implement + :meth:`tables`, :meth:`template_table`, and :meth:`slots`. See + :class:`TableFunctionUsage` -- keep ``operator_type`` and ``xfails`` + annotated ``ClassVar`` so the dataclass does not treat them as init fields. + """ + + operator_type: ClassVar[OperatorType] = OperatorType.AGGREGATE + + +class PredicateUsage(OperatorUsage): + """Extension point -- WHERE/ON/HAVING boolean predicates (INTERSECTS, etc.). + + Not yet populated. A predicate's patterns (in a ``WHERE``, in a join ``ON``, + negated, AND/OR-combined, ANY/ALL-quantified, literal-range vs. column + operand) are distinct from a table function's; populate this with a + predicate-specific ``canonical_patterns`` set and template table. See + :class:`TableFunctionUsage` for the descriptor shape and the + ``ClassVar`` caveat. + """ + + operator_type: ClassVar[OperatorType] = OperatorType.PREDICATE + + +class ScalarUsage(OperatorUsage): + """Extension point -- scalar functions usable in value expressions (DISTANCE). + + Not yet populated. A scalar's patterns (in the SELECT list, in a ``WHERE`` + comparison, in ``ORDER BY``, wrapped in an aggregate) are distinct from a + table function's; populate this with a scalar-specific ``canonical_patterns`` + set and template table. See :class:`TableFunctionUsage` for the descriptor + shape and the ``ClassVar`` caveat. + """ + + operator_type: ClassVar[OperatorType] = OperatorType.SCALAR + + +def templated_patterns(usage: OperatorUsage) -> tuple[UsagePattern, ...]: + """Return the canonical patterns :func:`render` can produce for ``usage``. + + Patterns are returned in canonical order, restricted to those the operator's + class has a template for. + """ + templates = usage.template_table() + return tuple( + pattern for pattern in usage.canonical_patterns if pattern in templates + ) + + +def render(pattern: UsagePattern, usage: OperatorUsage) -> str: + """Render the GIQL query that exercises ``pattern`` for ``usage``. + + :param pattern: + The usage pattern to render. + :param usage: + The operator's usage descriptor. + :return: + A complete GIQL query string. + :raises ValueError: + If the operator's class has no template for ``pattern`` -- for example + an unpopulated operator class. + """ + templates = usage.template_table() + if pattern not in templates: + raise ValueError( + f"{usage.name}: usage pattern {pattern.name} has no template for " + f"operator class {usage.operator_type.name}" + ) + return templates[pattern].format(**usage.slots()) + + +def rendered_cases(usage: OperatorUsage) -> list: + """Return rendered usage cases for ``usage`` as ``pytest.param`` values. + + Each param carries ``(pattern, query)``, a stable id, and a + ``pytest.mark.usage`` mark; patterns in ``usage.xfails`` additionally carry + a strict ``pytest.mark.xfail`` with the documented reason. The result is + ready to splat into ``@pytest.mark.parametrize(("pattern", "query"), ...)``. + """ + cases: list = [] + for pattern in templated_patterns(usage): + marks = [pytest.mark.usage(pattern)] + if pattern in usage.xfails: + marks.append( + pytest.mark.xfail(reason=usage.xfails[pattern], strict=True) + ) + cases.append( + pytest.param( + pattern, + render(pattern, usage), + id=f"{usage.name}-{pattern.name}", + marks=marks, + ) + ) + return cases + + +# --------------------------------------------------------------------------- +# Canonical fixture tables. +# +# The intervals are chosen so each pattern produces a non-trivial, +# discriminating result: the self-mode split, the reference-mode coverage +# filter, the overlap join, and the length/region filters all drop or +# transform some rows. The CANONICAL profile reuses these unchanged so its +# snapshot rows stay semantically identical to the originally committed +# manifests. +# --------------------------------------------------------------------------- +_FEATURES = TableFixture( + "features", + (("chr1", 0, 100, "a"), ("chr1", 60, 180, "b"), ("chr2", 10, 40, "c")), +) +_MASK = TableFixture( + "mask", + (("chr1", 0, 70, "m1"), ("chr1", 90, 200, "m2"), ("chr2", 0, 100, "m3")), +) +_ANNOTATIONS = TableFixture( + "annotations", + (("chr1", 0, 30, "g1"), ("chr2", 0, 1000, "g2")), +) + + +def _features_like(name: str, rows: tuple[tuple, ...]) -> TableFixture: + """Build a canonical-layout interval fixture with the given name and rows.""" + return TableFixture(name, rows) + + +# A small canonical annotations table reused by profiles whose own target / +# reference fixtures already isolate the behaviour under test; it backs the +# JOINED, SELF_JOIN_OVERLAP, and set-operation patterns. +def _annotations(rows: tuple[tuple, ...] = _ANNOTATIONS.rows) -> TableFixture: + """Build an ``annotations`` fixture (canonical layout) with the given rows.""" + return TableFixture("annotations", rows) + + +def _table_function_profile( + profile_id: str, + *, + features: tuple[tuple, ...], + annotations: tuple[tuple, ...] = _ANNOTATIONS.rows, + mask: tuple[tuple, ...] | None = None, + reference: bool = False, +) -> TableFunctionUsage: + """Build a canonical-layout DISJOIN profile from raw row tuples. + + When ``reference`` is true the profile is two-table-mode against a ``mask`` + table; otherwise it is self-mode. ``annotations`` backs the join and + set-operation patterns. + """ + feat_fixture = _features_like("features", features) + ann_fixture = _annotations(annotations) + if reference: + if mask is None: + raise ValueError(f"{profile_id}: reference-mode profile needs a mask") + return TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(feat_fixture, _features_like("mask", mask), ann_fixture), + reference="mask", + profile_id=profile_id, + ) + return TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(feat_fixture, ann_fixture), + profile_id=profile_id, + ) + + +# --------------------------------------------------------------------------- +# DISJOIN semantic profiles. +# +# Each profile is a TableFunctionUsage exercising one DISJOIN edge case. The +# functional suite parametrises every usage pattern over every profile and +# snapshots one manifest per (profile, mode). CANONICAL reuses the original +# self / reference fixtures so its rows match the first committed manifests. +# --------------------------------------------------------------------------- + +# CANONICAL -- the original benign data profile, self-mode and reference-mode. +DISJOIN_USAGE = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_FEATURES, _ANNOTATIONS), + profile_id="CANONICAL", +) +DISJOIN_REFERENCE_USAGE = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_FEATURES, _MASK, _ANNOTATIONS), + reference="mask", + profile_id="CANONICAL", +) + +# POINT_INTERVALS -- every target interval is zero-width (start == end). No +# cut produces a positive-length segment, so DISJOIN yields no rows. +_POINT_SELF = _table_function_profile( + "POINT_INTERVALS", + features=(("chr1", 10, 10, "p1"), ("chr1", 20, 20, "p2"), ("chr2", 5, 5, "p3")), +) +_POINT_REF = _table_function_profile( + "POINT_INTERVALS", + features=(("chr1", 10, 10, "p1"), ("chr1", 20, 20, "p2"), ("chr2", 5, 5, "p3")), + mask=(("chr1", 0, 100, "m1"), ("chr2", 0, 100, "m2")), + reference=True, +) + +# ADJACENT_INTERVALS -- intervals that touch end-to-start but never overlap. +# Self-mode cuts only at shared boundaries; segments stay whole. +_ADJACENT_SELF = _table_function_profile( + "ADJACENT_INTERVALS", + features=( + ("chr1", 0, 30, "a"), + ("chr1", 30, 60, "b"), + ("chr1", 60, 90, "c"), + ), +) +_ADJACENT_REF = _table_function_profile( + "ADJACENT_INTERVALS", + features=(("chr1", 0, 90, "t"),), + mask=(("chr1", 0, 30, "r1"), ("chr1", 30, 60, "r2"), ("chr1", 60, 90, "r3")), + reference=True, +) + +# NESTED_INTERVALS -- an interval fully contained in another. The inner +# interval's endpoints are interior cuts of the outer one. +_NESTED_SELF = _table_function_profile( + "NESTED_INTERVALS", + features=( + ("chr1", 0, 100, "outer"), + ("chr1", 30, 60, "inner"), + ("chr1", 40, 50, "innermost"), + ), +) +_NESTED_REF = _table_function_profile( + "NESTED_INTERVALS", + features=(("chr1", 0, 100, "outer"),), + mask=(("chr1", 30, 60, "inner"), ("chr1", 40, 50, "innermost")), + reference=True, +) + +# IDENTICAL_INTERVALS -- several rows share the exact same interval. DISJOIN +# partitions cuts per (chrom, start, end), so each duplicate is split +# independently and contributes its own sub-interval rows. +_IDENTICAL_SELF = _table_function_profile( + "IDENTICAL_INTERVALS", + features=( + ("chr1", 0, 50, "x1"), + ("chr1", 0, 50, "x2"), + ("chr1", 20, 80, "y"), + ), +) +_IDENTICAL_REF = _table_function_profile( + "IDENTICAL_INTERVALS", + features=(("chr1", 0, 50, "x1"), ("chr1", 0, 50, "x2")), + mask=(("chr1", 20, 80, "r1"), ("chr1", 20, 80, "r2")), + reference=True, +) + +# DUPLICATE_TARGETS -- duplicate target rows split against a single reference +# breakpoint set; each duplicate parent produces the same sub-intervals. +_DUPLICATE_SELF = _table_function_profile( + "DUPLICATE_TARGETS", + features=( + ("chr1", 0, 100, "d1"), + ("chr1", 0, 100, "d2"), + ("chr1", 0, 100, "d3"), + ("chr1", 40, 60, "cut"), + ), +) +_DUPLICATE_REF = _table_function_profile( + "DUPLICATE_TARGETS", + features=( + ("chr1", 0, 100, "d1"), + ("chr1", 0, 100, "d2"), + ("chr1", 0, 100, "d3"), + ), + mask=(("chr1", 40, 60, "r1"),), + reference=True, +) + +# MANY_CHROMOSOMES -- intervals spread across four chromosomes. No sub-interval +# may cross a chromosome boundary; cuts partition per chromosome. +_MANY_SELF = _table_function_profile( + "MANY_CHROMOSOMES", + features=( + ("chr1", 0, 100, "a"), + ("chr1", 50, 150, "b"), + ("chr2", 0, 60, "c"), + ("chr2", 30, 90, "d"), + ("chr3", 10, 40, "e"), + ("chrX", 5, 25, "f"), + ), + annotations=( + ("chr1", 0, 1000, "g1"), + ("chr2", 0, 1000, "g2"), + ("chr3", 0, 1000, "g3"), + ("chrX", 0, 1000, "g4"), + ), +) +_MANY_REF = _table_function_profile( + "MANY_CHROMOSOMES", + features=( + ("chr1", 0, 100, "a"), + ("chr2", 0, 60, "c"), + ("chr3", 10, 40, "e"), + ("chrX", 5, 25, "f"), + ), + mask=( + ("chr1", 50, 150, "r1"), + ("chr2", 30, 90, "r2"), + ("chr3", 0, 1000, "r3"), + ("chrX", 0, 1000, "r4"), + ), + annotations=( + ("chr1", 0, 1000, "g1"), + ("chr2", 0, 1000, "g2"), + ("chr3", 0, 1000, "g3"), + ("chrX", 0, 1000, "g4"), + ), + reference=True, +) + +# SINGLE_CHROMOSOME -- all intervals on one contig; a dense overlap stack. +_SINGLE_SELF = _table_function_profile( + "SINGLE_CHROMOSOME", + features=( + ("chr1", 0, 80, "a"), + ("chr1", 20, 100, "b"), + ("chr1", 40, 120, "c"), + ), + annotations=(("chr1", 0, 1000, "g1"),), +) +_SINGLE_REF = _table_function_profile( + "SINGLE_CHROMOSOME", + features=(("chr1", 0, 120, "t"),), + mask=(("chr1", 20, 60, "r1"), ("chr1", 60, 100, "r2")), + annotations=(("chr1", 0, 1000, "g1"),), + reference=True, +) + +# REFERENCE_GAP -- the reference set leaves a hole inside a target interval. +# DISJOIN cuts the target at the gap's edges, then the coverage filter drops +# the sub-interval lying over the uncovered gap. +_GAP_SELF = _table_function_profile( + "REFERENCE_GAP", + features=( + ("chr1", 0, 40, "a"), + ("chr1", 60, 100, "b"), + ("chr1", 0, 100, "spanning"), + ), +) +_GAP_REF = _table_function_profile( + "REFERENCE_GAP", + features=(("chr1", 0, 100, "t"),), + mask=(("chr1", 0, 40, "r1"), ("chr1", 60, 100, "r2")), + reference=True, +) + +# REFERENCE_SUPERSET -- the reference covers far more than the target. Every +# target sub-interval survives the coverage filter; cuts come from references +# interior to the target only. +_SUPERSET_SELF = _table_function_profile( + "REFERENCE_SUPERSET", + features=( + ("chr1", 20, 80, "a"), + ("chr1", 0, 200, "wide"), + ), + annotations=(("chr1", 0, 1000, "g1"),), +) +_SUPERSET_REF = _table_function_profile( + "REFERENCE_SUPERSET", + features=(("chr1", 40, 60, "t"),), + mask=(("chr1", 0, 1000, "r1"), ("chr1", 0, 500, "r2")), + annotations=(("chr1", 0, 1000, "g1"),), + reference=True, +) + +# EMPTY_TARGET -- the target table has no rows. DISJOIN yields nothing and +# must not error. +_EMPTY_TARGET_SELF = _table_function_profile( + "EMPTY_TARGET", + features=(), +) +_EMPTY_TARGET_REF = _table_function_profile( + "EMPTY_TARGET", + features=(), + mask=(("chr1", 0, 100, "r1"),), + reference=True, +) + +# EMPTY_REFERENCE -- the reference table has no rows, so there are no +# breakpoints and no coverage; DISJOIN yields nothing. +_EMPTY_REFERENCE_SELF = _table_function_profile( + "EMPTY_REFERENCE", + features=(), +) +_EMPTY_REFERENCE_REF = _table_function_profile( + "EMPTY_REFERENCE", + features=(("chr1", 0, 100, "t"), ("chr1", 50, 150, "u")), + mask=(), + reference=True, +) + +# CUSTOM_COLUMNS -- the target and reference store intervals under custom +# physical column names (seqid / lo / hi). The result must be identical to +# CANONICAL: the disjoin_* output columns are convention-fixed. +_CUSTOM_SCHEMA: tuple[tuple[str, str], ...] = ( + ("seqid", "VARCHAR"), + ("lo", "INTEGER"), + ("hi", "INTEGER"), + ("label", "VARCHAR"), +) +_CUSTOM_FEATURES = TableFixture( + "features", + (("chr1", 0, 100, "a"), ("chr1", 60, 180, "b"), ("chr2", 10, 40, "c")), + columns=_CUSTOM_SCHEMA, + table=Table("features", chrom_col="seqid", start_col="lo", end_col="hi"), +) +_CUSTOM_MASK = TableFixture( + "mask", + (("chr1", 0, 70, "m1"), ("chr1", 90, 200, "m2"), ("chr2", 0, 100, "m3")), + columns=_CUSTOM_SCHEMA, + table=Table("mask", chrom_col="seqid", start_col="lo", end_col="hi"), +) +_CUSTOM_SELF = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_CUSTOM_FEATURES, _ANNOTATIONS), + profile_id="CUSTOM_COLUMNS", +) +_CUSTOM_REF = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_CUSTOM_FEATURES, _CUSTOM_MASK, _ANNOTATIONS), + reference="mask", + profile_id="CUSTOM_COLUMNS", +) + +# ONE_BASED_CLOSED_TARGET -- the target is stored 1-based closed. GIQL must +# canonicalize endpoints; the disjoin_* sub-intervals come back 0-based +# half-open. The 1-based-closed encoding of canonical [0,100) is [1,100], +# of [60,180) is [61,180], of [10,40) is [11,40]. +_ONE_BASED_FEATURES = TableFixture( + "features", + (("chr1", 1, 100, "a"), ("chr1", 61, 180, "b"), ("chr2", 11, 40, "c")), + table=Table("features", coordinate_system="1based", interval_type="closed"), +) +_ONE_BASED_SELF = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_ONE_BASED_FEATURES, _ANNOTATIONS), + profile_id="ONE_BASED_CLOSED_TARGET", +) +# 1-based-closed mask encoding of canonical [0,70)->[1,70], [90,200)->[91,200], +# [0,100)->[1,100]. +_ONE_BASED_MASK = TableFixture( + "mask", + (("chr1", 1, 70, "m1"), ("chr1", 91, 200, "m2"), ("chr2", 1, 100, "m3")), + table=Table("mask", coordinate_system="1based", interval_type="closed"), +) +_ONE_BASED_REF = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_ONE_BASED_FEATURES, _ONE_BASED_MASK, _ANNOTATIONS), + reference="mask", + profile_id="ONE_BASED_CLOSED_TARGET", +) + +# MIXED_ENCODING -- target and reference use different conventions. The target +# is 0-based half-open; the reference is 1-based closed. Both canonicalize to +# the same 0-based half-open space, so the result matches CANONICAL's. +_MIXED_FEATURES = _FEATURES # canonical 0-based half-open +_MIXED_MASK = TableFixture( + "mask", + (("chr1", 1, 70, "m1"), ("chr1", 91, 200, "m2"), ("chr2", 1, 100, "m3")), + table=Table("mask", coordinate_system="1based", interval_type="closed"), +) +_MIXED_SELF = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_MIXED_FEATURES, _ANNOTATIONS), + profile_id="MIXED_ENCODING", +) +_MIXED_REF = TableFunctionUsage( + name="DISJOIN", + target="features", + out_chrom="disjoin_chrom", + out_start="disjoin_start", + out_end="disjoin_end", + fixtures=(_MIXED_FEATURES, _MIXED_MASK, _ANNOTATIONS), + reference="mask", + profile_id="MIXED_ENCODING", +) + + +@dataclass(frozen=True) +class DisjoinProfile: + """A named DISJOIN semantic profile: its self-mode and reference-mode usages. + + Each profile binds one edge-case fixture set to both a self-mode and a + reference-mode :class:`TableFunctionUsage`, so the functional suite can + parametrise every usage pattern over the same data in both modes. + """ + + profile_id: str + self_usage: TableFunctionUsage + reference_usage: TableFunctionUsage + + +# The full DISJOIN profile matrix. Profile ids are unique -- they prefix the +# manifest keys, so a collision would silently overwrite snapshot rows. +DISJOIN_PROFILES: tuple[DisjoinProfile, ...] = ( + DisjoinProfile("CANONICAL", DISJOIN_USAGE, DISJOIN_REFERENCE_USAGE), + DisjoinProfile("POINT_INTERVALS", _POINT_SELF, _POINT_REF), + DisjoinProfile("ADJACENT_INTERVALS", _ADJACENT_SELF, _ADJACENT_REF), + DisjoinProfile("NESTED_INTERVALS", _NESTED_SELF, _NESTED_REF), + DisjoinProfile("IDENTICAL_INTERVALS", _IDENTICAL_SELF, _IDENTICAL_REF), + DisjoinProfile("DUPLICATE_TARGETS", _DUPLICATE_SELF, _DUPLICATE_REF), + DisjoinProfile("MANY_CHROMOSOMES", _MANY_SELF, _MANY_REF), + DisjoinProfile("SINGLE_CHROMOSOME", _SINGLE_SELF, _SINGLE_REF), + DisjoinProfile("REFERENCE_GAP", _GAP_SELF, _GAP_REF), + DisjoinProfile("REFERENCE_SUPERSET", _SUPERSET_SELF, _SUPERSET_REF), + DisjoinProfile("EMPTY_TARGET", _EMPTY_TARGET_SELF, _EMPTY_TARGET_REF), + DisjoinProfile("EMPTY_REFERENCE", _EMPTY_REFERENCE_SELF, _EMPTY_REFERENCE_REF), + DisjoinProfile("CUSTOM_COLUMNS", _CUSTOM_SELF, _CUSTOM_REF), + DisjoinProfile("ONE_BASED_CLOSED_TARGET", _ONE_BASED_SELF, _ONE_BASED_REF), + DisjoinProfile("MIXED_ENCODING", _MIXED_SELF, _MIXED_REF), +) From b6cb344aac817eb7cdbf124130d22d1098080756 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 16:05:40 -0400 Subject: [PATCH 006/142] fix: Reject malformed DISJOIN argument lists DISJOIN bound only the first positional argument and silently discarded any extras, so DISJOIN(features, refs) dropped the intended reference set without warning. Unknown named arguments were passed through unchecked as well. from_arg_list now raises a ParseError naming the single-positional-arg limit and rejecting any named argument outside the DISJOIN schema, so a mistyped or misplaced reference fails loudly at parse time. --- src/giql/expressions.py | 13 +++++++++++++ tests/test_disjoin_parsing.py | 36 +++++++++++++++++++++++++---------- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index c414dac..747d63b 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -4,6 +4,7 @@ """ from sqlglot import exp +from sqlglot.errors import ParseError class GenomicRange(exp.Expression): @@ -228,6 +229,18 @@ class GIQLDisjoin(exp.Func): @classmethod def from_arg_list(cls, args): kwargs, positional_args = _split_named_and_positional(args) + unknown = set(kwargs) - set(cls.arg_types) + if unknown: + raise ParseError( + f"DISJOIN got unexpected named argument(s): " + f"{', '.join(sorted(unknown))}. Valid arguments: reference." + ) + if len(positional_args) > 1: + raise ParseError( + "DISJOIN accepts at most one positional argument (the target " + f"table); got {len(positional_args)}. Pass the reference set " + "as 'reference := ...'." + ) if len(positional_args) >= 1: kwargs["this"] = positional_args[0] return cls(**kwargs) diff --git a/tests/test_disjoin_parsing.py b/tests/test_disjoin_parsing.py index 5480445..a67a68c 100644 --- a/tests/test_disjoin_parsing.py +++ b/tests/test_disjoin_parsing.py @@ -4,8 +4,10 @@ and maps positional and named arguments onto the GIQLDisjoin AST node. """ +import pytest from sqlglot import exp from sqlglot import parse_one +from sqlglot.errors import ParseError from giql.dialect import GIQLDialect from giql.expressions import GIQLDisjoin @@ -69,23 +71,37 @@ def test_from_arg_list_should_leave_reference_unset_when_omitted(self): # Assert assert node.args.get("reference") is None - def test_from_arg_list_should_ignore_extra_positional_argument(self): - """Test that a bare second positional argument is not bound. + def test_from_arg_list_should_reject_extra_positional_argument(self): + """Test that a bare second positional argument is rejected. Given: - A GIQL query with DISJOIN(features, refs) using a bare second - positional argument. + A GIQL query with DISJOIN(features, refs) passing the reference + set as a bare second positional argument. When: Parsing the query. Then: - It should bind `this` to features and leave `reference` unset. + It should raise a ParseError naming the positional-argument limit. """ - # Arrange & act - node = _disjoin_node("SELECT * FROM DISJOIN(features, refs)") + # Arrange, act, & assert + with pytest.raises(ParseError, match="at most one positional"): + parse_one("SELECT * FROM DISJOIN(features, refs)", dialect=GIQLDialect) - # Assert - assert node.this.name == "features" - assert node.args.get("reference") is None + def test_from_arg_list_should_reject_unknown_named_argument(self): + """Test that an unrecognized named argument is rejected. + + Given: + A GIQL query with DISJOIN(features, mask := refs) naming an + argument that is not part of the DISJOIN schema. + When: + Parsing the query. + Then: + It should raise a ParseError naming the unexpected argument. + """ + # Arrange, act, & assert + with pytest.raises(ParseError, match="unexpected named argument"): + parse_one( + "SELECT * FROM DISJOIN(features, mask := refs)", dialect=GIQLDialect + ) def test_from_arg_list_should_map_reference_when_named_with_walrus(self): """Test that a walrus-named reference argument is carried. From 13ed590137da9c26e48d1f53b1ba2506340df848 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 16:05:48 -0400 Subject: [PATCH 007/142] fix: Resolve DISJOIN references with correct CTE scoping A bare DISJOIN reference name was matched against registered tables first and, failing that, assumed to be a canonical CTE. A CTE sharing a registered table's name therefore silently inherited that table's coordinate system, and a name matching neither produced SQL referencing a relation that does not exist. Reference resolution now checks enclosing WITH clauses first, letting an in-query CTE shadow a registered table the way SQL scoping does, and raises a ValueError when a bare name matches neither a CTE nor a registered table. Bare SELECT and UNION references are accepted alongside Subquery nodes, and a reference using the reserved __giql_dj_ prefix is rejected since that prefix names the operator's internal CTEs. --- src/giql/generators/base.py | 86 +++++++++++++++++++++++------ tests/test_disjoin_transpilation.py | 70 +++++++++++++++++++++++ 2 files changed, 138 insertions(+), 18 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index e53a5f5..a742065 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -294,7 +294,7 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: target_name, (target_chrom, target_start, target_end) = ( self._resolve_target_table(expression) ) - target_table = self.tables.get(target_name) if self.tables else None + target_table = self.tables.get(target_name) # Resolve the reference relation (defaults to the target set). ref_from, ref_chrom, ref_start, ref_end, ref_table = ( @@ -900,11 +900,12 @@ def _resolve_disjoin_reference( The reference contributes the breakpoints at which target intervals are cut. When ``reference`` is omitted it defaults to the target set. - A bare reference name is resolved against the registered tables - first: a match contributes that table's column names and coordinate - system. A name with no registered table is treated as a CTE and - assumed canonical. A CTE that shares a registered table's name will - therefore inherit that table's configuration -- avoid such collisions. + A bare reference name is resolved against the query's CTEs first: a + name defined by an enclosing ``WITH`` clause shadows a registered table + of the same name (matching SQL scoping) and is assumed to expose + canonical 0-based half-open ``chrom`` / ``start`` / ``end`` columns. A + name with no such CTE but a registered table contributes that table's + column names and coordinate system. A name that is neither is an error. :param expression: GIQLDisjoin expression node @@ -917,9 +918,12 @@ def _resolve_disjoin_reference( :return: Tuple of ``(from_clause, chrom_col, start_col, end_col, table)`` where ``from_clause`` is the text following ``FROM`` inside the - ``__giql_dj_ref`` CTE. A subquery or unregistered (CTE) reference - is assumed to expose canonical 0-based half-open ``chrom`` / - ``start`` / ``end`` columns, so ``table`` is ``None`` for those. + ``__giql_dj_ref`` CTE. A subquery or CTE reference is assumed to + expose canonical 0-based half-open ``chrom`` / ``start`` / ``end`` + columns, so ``table`` is ``None`` for those. + :raises ValueError: + If the reference is not a table name, a CTE, or a subquery, or if + a bare name resolves to neither a registered table nor a CTE. """ reference = expression.args.get("reference") tc, ts, te = target_cols @@ -929,7 +933,7 @@ def _resolve_disjoin_reference( return target_name, tc, ts, te, target_table # Subquery reference: inline it as an aliased derived table. - if isinstance(reference, exp.Subquery): + if isinstance(reference, (exp.Subquery, exp.Select, exp.Union)): from_clause = f"{self.sql(reference)} AS __giql_dj_rs" return ( from_clause, @@ -939,13 +943,36 @@ def _resolve_disjoin_reference( None, ) - # Bare table or CTE name. - if isinstance(reference, (exp.Table, exp.Column)): - ref_name = reference.name - else: - ref_name = self.sql(reference).strip('"') + # Anything that is not a bare table/CTE name is unsupported. + if not isinstance(reference, (exp.Table, exp.Column, exp.Identifier)): + raise ValueError( + "DISJOIN reference must be a table name, a CTE, or a " + f"(SELECT ...) subquery; got {type(reference).__name__}: " + f"{reference}" + ) + + ref_name = reference.name - ref_table = self.tables.get(ref_name) if self.tables else None + # The __giql_dj_ prefix names the operator's internal CTEs. + if ref_name.startswith("__giql_dj_"): + raise ValueError( + f"DISJOIN reference {ref_name!r} uses the reserved " + "'__giql_dj_' prefix, which names the operator's internal " + "CTEs. Rename the reference relation." + ) + + # A CTE from an enclosing WITH shadows a registered table of the same + # name; it is assumed to expose canonical default columns. + if ref_name in self._enclosing_cte_names(expression): + return ( + ref_name, + DEFAULT_CHROM_COL, + DEFAULT_START_COL, + DEFAULT_END_COL, + None, + ) + + ref_table = self.tables.get(ref_name) if ref_table: return ( ref_name, @@ -954,8 +981,31 @@ def _resolve_disjoin_reference( ref_table.end_col, ref_table, ) - # Unregistered name (e.g. a CTE): assume canonical default columns. - return ref_name, DEFAULT_CHROM_COL, DEFAULT_START_COL, DEFAULT_END_COL, None + raise ValueError( + f"DISJOIN reference {ref_name!r} is neither a registered table " + "nor a CTE defined in this query. Register the table or define " + "the CTE before transpiling." + ) + + @staticmethod + def _enclosing_cte_names(expression: exp.Expression) -> set[str]: + """Collect CTE names visible to an expression from enclosing WITH clauses. + + Walks the ancestor chain and gathers the aliases of every ``WITH``-clause + CTE, so a DISJOIN reference can be recognized as an in-query CTE rather + than a registered table. + """ + names: set[str] = set() + node: exp.Expression | None = expression + while node is not None: + if isinstance(node, exp.Select): + with_clause = node.args.get("with_") or node.args.get("with") + if with_clause is not None: + for cte in with_clause.expressions: + if cte.alias: + names.add(cte.alias) + node = node.parent + return names def _get_column_refs( self, diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index 6844ee8..eb37e15 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -5,6 +5,8 @@ canonicalization are reflected in the generated SQL. """ +import pytest + from giql import transpile from giql.table import Table @@ -125,3 +127,71 @@ def test_giqldisjoin_sql_should_inline_subquery_reference(self): ) assert "AS __giql_dj_rs" in sql + + def test_giqldisjoin_sql_should_resolve_in_query_cte_reference(self): + """ + GIVEN a DISJOIN reference naming a CTE defined in the outer query + WHEN transpiling to SQL + THEN the reference CTE should select from that CTE name + """ + sql = transpile( + "WITH bins AS (SELECT 1) " + "SELECT * FROM DISJOIN(features, reference := bins)", + tables=["features"], + ) + + assert "__giql_dj_ref AS (SELECT * FROM bins)" in sql + + def test_giqldisjoin_sql_should_let_cte_shadow_registered_table(self): + """ + GIVEN a reference name shared by a registered 1-based table and a CTE + WHEN transpiling to SQL + THEN the CTE should shadow the table and canonical columns stay unshifted + """ + sql = transpile( + "WITH refs AS (SELECT 1) " + "SELECT * FROM DISJOIN(features, reference := refs)", + tables=[ + "features", + Table("refs", coordinate_system="1based", interval_type="closed"), + ], + ) + + assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql + assert '("start" - 1)' not in sql + + def test_giqldisjoin_sql_should_reject_literal_range_reference(self): + """ + GIVEN a DISJOIN call whose reference is a literal genomic range + WHEN transpiling to SQL + THEN transpilation should raise an error rejecting the reference form + """ + with pytest.raises(ValueError, match="DISJOIN reference must be"): + transpile( + "SELECT * FROM DISJOIN(features, reference := 'chr1:1-9')", + tables=["features"], + ) + + def test_giqldisjoin_sql_should_reject_reserved_prefix_reference(self): + """ + GIVEN a DISJOIN reference name using the reserved __giql_dj_ prefix + WHEN transpiling to SQL + THEN transpilation should raise an error naming the reserved prefix + """ + with pytest.raises(ValueError, match="reserved"): + transpile( + "SELECT * FROM DISJOIN(features, reference := __giql_dj_ref)", + tables=["features"], + ) + + def test_giqldisjoin_sql_should_reject_unknown_reference_name(self): + """ + GIVEN a DISJOIN reference that is neither a registered table nor a CTE + WHEN transpiling to SQL + THEN transpilation should raise an error rejecting the unknown name + """ + with pytest.raises(ValueError, match="neither a registered table"): + transpile( + "SELECT * FROM DISJOIN(features, reference := missing)", + tables=["features"], + ) From 00444511a409ef37900a17f13955e55253edec79 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 16:05:52 -0400 Subject: [PATCH 008/142] test: Run DISJOIN execution tests on SQLite and DuckDB The end-to-end DISJOIN suite ran against DuckDB only. Parametrize it over a run fixture so every execution test also runs on in-memory SQLite, skipping SQLite below 3.25 where the LEAD() window function DISJOIN emits is unavailable. --- tests/test_disjoin_udf.py | 93 ++++++++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 30 deletions(-) diff --git a/tests/test_disjoin_udf.py b/tests/test_disjoin_udf.py index 48b5d7a..f55803d 100644 --- a/tests/test_disjoin_udf.py +++ b/tests/test_disjoin_udf.py @@ -1,39 +1,72 @@ -"""End-to-end execution tests for the DISJOIN operator on DuckDB. +"""End-to-end execution tests for the DISJOIN operator. -Tests transpile DISJOIN queries and execute them against in-memory DuckDB -to verify split correctness, the coverage filter, parent passthrough, and -degenerate-input handling. +Tests transpile DISJOIN queries and execute them against in-memory DuckDB and +SQLite to verify split correctness, the coverage filter, parent passthrough, +and degenerate-input handling on every supported in-process backend. """ +import sqlite3 + import duckdb +import pytest from giql import transpile +_TABLE_COLUMNS = '(chrom VARCHAR, "start" INTEGER, "end" INTEGER, name VARCHAR)' + -def _run(query: str, tables: list[str], **table_data): +def _run_duckdb(query: str, tables: list[str], **table_data): """Transpile a DISJOIN query and execute it against in-memory DuckDB.""" conn = duckdb.connect(":memory:") for name, rows in table_data.items(): - conn.execute( - f'CREATE TABLE {name}(chrom VARCHAR, "start" INTEGER, ' - f'"end" INTEGER, name VARCHAR)' - ) + conn.execute(f"CREATE TABLE {name}{_TABLE_COLUMNS}") + conn.executemany(f"INSERT INTO {name} VALUES (?, ?, ?, ?)", rows) + result = conn.execute(transpile(query, tables=tables)).fetchall() + conn.close() + return result + + +def _run_sqlite(query: str, tables: list[str], **table_data): + """Transpile a DISJOIN query and execute it against in-memory SQLite.""" + conn = sqlite3.connect(":memory:") + for name, rows in table_data.items(): + conn.execute(f"CREATE TABLE {name}{_TABLE_COLUMNS}") conn.executemany(f"INSERT INTO {name} VALUES (?, ?, ?, ?)", rows) result = conn.execute(transpile(query, tables=tables)).fetchall() conn.close() return result +_ENGINES = {"duckdb": _run_duckdb, "sqlite": _run_sqlite} + + +@pytest.fixture( + params=[ + "duckdb", + pytest.param( + "sqlite", + marks=pytest.mark.skipif( + sqlite3.sqlite_version_info < (3, 25), + reason="DISJOIN emits LEAD(), which requires SQLite 3.25+", + ), + ), + ] +) +def run(request): + """Yield an engine-specific transpile-and-execute callable.""" + return _ENGINES[request.param] + + class TestDisjoinExecution: - """End-to-end DISJOIN correctness on DuckDB.""" + """End-to-end DISJOIN correctness on every supported backend.""" - def test_disjoin_should_split_intervals_at_breakpoints(self): + def test_disjoin_should_split_intervals_at_breakpoints(self, run): """ GIVEN two overlapping target intervals A=[0,20) and B=[10,30) WHEN DISJOIN runs in self-mode THEN each interval should be split at the other's interior breakpoint """ - result = _run( + result = run( "SELECT name, disjoin_start, disjoin_end FROM DISJOIN(features) " "ORDER BY name, disjoin_start", tables=["features"], @@ -47,13 +80,13 @@ def test_disjoin_should_split_intervals_at_breakpoints(self): ("B", 20, 30), ] - def test_disjoin_should_yield_bioconductor_partition_when_distinct(self): + def test_disjoin_should_yield_bioconductor_partition_when_distinct(self, run): """ GIVEN overlapping target intervals in self-mode WHEN selecting DISTINCT sub-intervals from DISJOIN THEN the result should be the globally non-overlapping partition """ - result = _run( + result = run( "SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end " "FROM DISJOIN(features) ORDER BY disjoin_start", tables=["features"], @@ -62,13 +95,13 @@ def test_disjoin_should_yield_bioconductor_partition_when_distinct(self): assert result == [("chr1", 0, 10), ("chr1", 10, 20), ("chr1", 20, 30)] - def test_disjoin_should_split_against_explicit_reference(self): + def test_disjoin_should_split_against_explicit_reference(self, run): """ GIVEN a target interval and a separate reference set WHEN DISJOIN runs with an explicit reference THEN the target should be cut at the reference breakpoints """ - result = _run( + result = run( "SELECT disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", tables=["features", "refs"], @@ -78,13 +111,13 @@ def test_disjoin_should_split_against_explicit_reference(self): assert result == [(0, 10), (10, 30)] - def test_disjoin_should_drop_pieces_overlapping_no_reference(self): + def test_disjoin_should_drop_pieces_overlapping_no_reference(self, run): """ GIVEN a target spanning a gap in the reference coverage WHEN DISJOIN runs with an explicit reference THEN sub-intervals overlapping no reference interval should be dropped """ - result = _run( + result = run( "SELECT disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", tables=["features", "refs"], @@ -94,13 +127,13 @@ def test_disjoin_should_drop_pieces_overlapping_no_reference(self): assert result == [(0, 10), (20, 30)] - def test_disjoin_should_yield_no_rows_when_target_is_a_point(self): + def test_disjoin_should_yield_no_rows_when_target_is_a_point(self, run): """ GIVEN a zero-length target interval WHEN DISJOIN runs THEN it should produce no output rows """ - result = _run( + result = run( "SELECT * FROM DISJOIN(features)", tables=["features"], features=[("chr1", 5, 5, "P")], @@ -108,13 +141,13 @@ def test_disjoin_should_yield_no_rows_when_target_is_a_point(self): assert result == [] - def test_disjoin_should_split_duplicate_targets_independently(self): + def test_disjoin_should_split_duplicate_targets_independently(self, run): """ GIVEN two target rows with identical geometry WHEN DISJOIN runs with a reference that cuts them THEN each duplicate row should be split independently """ - result = _run( + result = run( "SELECT name, disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs) ORDER BY name, disjoin_start", tables=["features", "refs"], @@ -124,13 +157,13 @@ def test_disjoin_should_split_duplicate_targets_independently(self): assert result == [("X", 0, 5), ("X", 5, 10), ("Y", 0, 5), ("Y", 5, 10)] - def test_disjoin_should_not_cross_chromosome_boundaries(self): + def test_disjoin_should_not_cross_chromosome_boundaries(self, run): """ GIVEN target intervals on different chromosomes WHEN DISJOIN runs in self-mode THEN sub-intervals should never span a chromosome boundary """ - result = _run( + result = run( "SELECT disjoin_chrom, disjoin_start, disjoin_end FROM DISJOIN(features) " "ORDER BY disjoin_chrom, disjoin_start", tables=["features"], @@ -139,14 +172,14 @@ def test_disjoin_should_not_cross_chromosome_boundaries(self): assert result == [("chr1", 0, 20), ("chr2", 5, 25)] - def test_disjoin_should_pass_through_non_interval_columns(self): + def test_disjoin_should_pass_through_non_interval_columns(self, run): """ GIVEN a target table carrying a non-interval column WHEN DISJOIN runs THEN every output row should carry the intact parent row alongside its sub-interval """ - result = _run( + result = run( 'SELECT name, chrom, "start", "end", disjoin_start, disjoin_end ' "FROM DISJOIN(features) ORDER BY disjoin_start, name", tables=["features"], @@ -160,13 +193,13 @@ def test_disjoin_should_pass_through_non_interval_columns(self): ("B", "chr1", 10, 30, 20, 30), ] - def test_disjoin_should_accept_a_cte_as_reference(self): + def test_disjoin_should_accept_a_cte_as_reference(self, run): """ GIVEN a reference supplied as a CTE defined in the outer query WHEN DISJOIN runs against that CTE THEN the target should be split at the CTE's breakpoints """ - result = _run( + result = run( "WITH bins AS (" " SELECT 'chr1' AS chrom, 0 AS \"start\", 10 AS \"end\" " " UNION ALL SELECT 'chr1', 10, 20" @@ -179,13 +212,13 @@ def test_disjoin_should_accept_a_cte_as_reference(self): assert result == [(0, 10), (10, 20)] - def test_disjoin_should_not_cut_at_breakpoint_on_target_boundary(self): + def test_disjoin_should_not_cut_at_breakpoint_on_target_boundary(self, run): """ GIVEN a reference breakpoint coinciding with a target boundary WHEN DISJOIN runs THEN the target should not be split at that boundary """ - result = _run( + result = run( "SELECT disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs)", tables=["features", "refs"], From a013d71386a2cc7c8a2859b892f4982cfb53b9b1 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 16:05:57 -0400 Subject: [PATCH 009/142] docs: Move DISJOIN into a dedicated Set Operations page DISJOIN was documented under the aggregation operators, but it multiplies rows rather than aggregating them. Move its reference into a new set-operators page, list the page in the dialect index and toctree, and point the MCP operator catalog at the relocated doc. --- docs/dialect/aggregation-operators.rst | 129 -------------------- docs/dialect/index.rst | 19 +++ docs/dialect/set-operators.rst | 157 +++++++++++++++++++++++++ docs/index.rst | 1 + src/giql/mcp/server.py | 5 +- 5 files changed, 180 insertions(+), 131 deletions(-) create mode 100644 docs/dialect/set-operators.rst diff --git a/docs/dialect/aggregation-operators.rst b/docs/dialect/aggregation-operators.rst index 2d488bf..9887b87 100644 --- a/docs/dialect/aggregation-operators.rst +++ b/docs/dialect/aggregation-operators.rst @@ -329,132 +329,3 @@ Related Operators - :ref:`CLUSTER ` - Assign cluster IDs without merging - :ref:`INTERSECTS ` - Test for overlap between specific pairs - ----- - -.. _disjoin-operator: - -DISJOIN -------- - -Split genomic intervals at reference breakpoints into sub-intervals that -never partially overlap a reference interval. - -Description -~~~~~~~~~~~ - -The ``DISJOIN`` operator is a table function: it takes a set of *target* -intervals and a *reference* set of intervals, and cuts each target interval -at every reference breakpoint (a reference ``start`` or ``end`` position) -that falls strictly inside it. Every resulting sub-interval is fully -contained by each reference interval it overlaps -- it can never *partially* -overlap one. - -Unlike :ref:`MERGE ` and :ref:`CLUSTER `, -which aggregate intervals, ``DISJOIN`` *multiplies* rows: one target interval -becomes one or more sub-interval rows. The full target row passes through -unchanged and the sub-interval is appended as ``disjoin_chrom``, -``disjoin_start``, ``disjoin_end``. - -When no ``reference`` is given it defaults to the target set, so -``DISJOIN(features)`` splits the set against its own breakpoints. Selecting -the distinct sub-intervals then yields the globally non-overlapping partition --- the equivalent of Bioconductor's ``GenomicRanges::disjoin()``. - -This is useful for: - -- Partitioning a set of intervals into non-overlapping segments -- Re-tiling target features onto a data-defined or external grid -- Splitting features so downstream aggregates never double-count overlaps - -Syntax -~~~~~~ - -.. code-block:: sql - - -- Self-mode: split the set against its own breakpoints - SELECT * FROM DISJOIN(features) - - -- Split target features against an explicit reference set - SELECT * FROM DISJOIN(features, reference := mask) - - -- The reference may be a subquery - SELECT * FROM DISJOIN(features, reference := (SELECT * FROM mask)) - -Parameters -~~~~~~~~~~ - -**target** - The table of intervals to split. - -**reference** *(optional)* - A table, CTE, or subquery whose interval boundaries supply the breakpoints. - Defaults to ``target`` when omitted. - -Return Value -~~~~~~~~~~~~ - -Every column of the matched target row, passed through unchanged, plus the -sub-interval: - -- ``disjoin_chrom`` - Chromosome of the sub-interval -- ``disjoin_start`` - Start of the sub-interval -- ``disjoin_end`` - End of the sub-interval - -A sub-interval that overlaps no reference interval is dropped (the coverage -filter). In self-mode every sub-interval is covered by its own parent, so -nothing is dropped. - -Examples -~~~~~~~~ - -**Partition a set of intervals:** - -Given two overlapping intervals ``A = [0, 20)`` and ``B = [10, 30)``, -``DISJOIN`` in self-mode cuts ``A`` at breakpoint ``10`` and ``B`` at -breakpoint ``20``: - -.. code-block:: sql - - SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end - FROM DISJOIN(features) - ORDER BY disjoin_start - - -- Returns the partition: [0,10), [10,20), [20,30) - -**Split features against a mask:** - -.. code-block:: sql - - SELECT name, disjoin_start, disjoin_end - FROM DISJOIN(features, reference := blacklist) - -**Re-tile against a uniform grid:** - -.. code-block:: sql - - WITH bins AS ( - SELECT 'chr1' AS chrom, x AS start, x + 1000 AS "end" - FROM range(0, 250000000, 1000) AS t(x) - ) - SELECT * FROM DISJOIN(features, reference := bins) - -.. note:: - - ``DISJOIN`` is a table function and appears in the ``FROM`` clause, like - ``NEAREST``. On PostgreSQL the derived table must be given an alias - (``FROM DISJOIN(features) AS d``). - -.. note:: - - The appended ``disjoin_start`` / ``disjoin_end`` columns are emitted in the - target table's coordinate system -- the same convention as its - passed-through ``start`` / ``end`` columns, so every column of an output row - shares one convention. Cut positions are computed canonically inside the - operator; only their final representation follows the target table. - -Related Operators -~~~~~~~~~~~~~~~~~ - -- :ref:`MERGE ` - Collapse overlapping intervals into one -- :ref:`CLUSTER ` - Assign cluster IDs without splitting diff --git a/docs/dialect/index.rst b/docs/dialect/index.rst index 8d70e9d..dad1e3d 100644 --- a/docs/dialect/index.rst +++ b/docs/dialect/index.rst @@ -98,6 +98,24 @@ Combine and cluster genomic intervals. See :doc:`aggregation-operators` for detailed documentation. +Set Operations +-------------- + +Split genomic intervals against another set of intervals. + +.. list-table:: + :header-rows: 1 + :widths: 20 50 30 + + * - Operator + - Description + - Example + * - :ref:`DISJOIN ` + - Split intervals at reference breakpoints into sub-intervals + - ``SELECT * FROM DISJOIN(features, reference := mask)`` + +See :doc:`set-operators` for detailed documentation. + Set Quantifiers --------------- @@ -127,4 +145,5 @@ See :doc:`quantifiers` for detailed documentation. .. spatial-operators .. distance-operators .. aggregation-operators +.. set-operators .. quantifiers diff --git a/docs/dialect/set-operators.rst b/docs/dialect/set-operators.rst new file mode 100644 index 0000000..abf38ff --- /dev/null +++ b/docs/dialect/set-operators.rst @@ -0,0 +1,157 @@ +Set Operations +============== + +Set operations cut a set of genomic intervals against another set of intervals, +producing finer-grained sub-intervals. Unlike the aggregation operators, which +collapse intervals into summarized regions, a set operation *multiplies* rows -- +one input interval becomes one or more output rows. + +.. _disjoin-operator: + +DISJOIN +------- + +Split genomic intervals at reference breakpoints into sub-intervals that +never partially overlap a reference interval. + +Description +~~~~~~~~~~~ + +The ``DISJOIN`` operator is a table function: it takes a set of *target* +intervals and a *reference* set of intervals, and cuts each target interval +at every reference breakpoint (a reference ``start`` or ``end`` position) +that falls strictly inside it. Every resulting sub-interval is fully +contained by each reference interval it overlaps -- it can never *partially* +overlap one. + +Unlike :ref:`MERGE ` and :ref:`CLUSTER `, +which aggregate intervals, ``DISJOIN`` *multiplies* rows: one target interval +becomes one or more sub-interval rows. The full target row passes through +unchanged and the sub-interval is appended as ``disjoin_chrom``, +``disjoin_start``, ``disjoin_end``. + +When no ``reference`` is given it defaults to the target set, so +``DISJOIN(features)`` splits the set against its own breakpoints. Selecting +the distinct sub-intervals then yields the globally non-overlapping partition +-- the equivalent of Bioconductor's ``GenomicRanges::disjoin()``. ``DISJOIN`` +deliberately departs from that set-reducing shape: as a SQL table function it +*multiplies* rows and keeps each parent row intact, so it composes with +``JOIN`` and ``SELECT``. ``SELECT DISTINCT disjoin_chrom, disjoin_start, +disjoin_end`` recovers the canonical ``disjoin()`` partition exactly. + +This is useful for: + +- Partitioning a set of intervals into non-overlapping segments +- Re-tiling target features onto a data-defined or external grid +- Splitting features so downstream aggregates never double-count overlaps + +Syntax +~~~~~~ + +.. code-block:: sql + + -- Self-mode: split the set against its own breakpoints + SELECT * FROM DISJOIN(features) + + -- Split target features against an explicit reference set + SELECT * FROM DISJOIN(features, reference := mask) + + -- The reference may be a subquery + SELECT * FROM DISJOIN(features, reference := (SELECT * FROM mask)) + +Parameters +~~~~~~~~~~ + +**target** + The table of intervals to split. Must be a table registered with the + transpiler so its genomic columns can be resolved. + +**reference** *(optional)* + A registered table, a CTE defined in the same query, or a ``(SELECT ...)`` + subquery whose interval boundaries supply the breakpoints. Defaults to + ``target`` when omitted. A bare name that is neither a registered table nor + an in-query CTE is rejected. + +Return Value +~~~~~~~~~~~~ + +Every column of the matched target row, passed through unchanged, plus the +sub-interval: + +- ``disjoin_chrom`` - Chromosome of the sub-interval +- ``disjoin_start`` - Start of the sub-interval +- ``disjoin_end`` - End of the sub-interval + +A sub-interval that overlaps no reference interval is dropped (the coverage +filter). In self-mode every sub-interval is covered by its own parent, so +nothing is dropped. In reference mode a target interval that overlaps no +reference interval at all yields no output rows -- ``DISJOIN`` can drop a +target entirely, not merely trim it. + +Examples +~~~~~~~~ + +**Partition a set of intervals:** + +Given two overlapping intervals ``A = [0, 20)`` and ``B = [10, 30)``, +``DISJOIN`` in self-mode cuts ``A`` at breakpoint ``10`` and ``B`` at +breakpoint ``20``: + +.. code-block:: sql + + SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end + FROM DISJOIN(features) + ORDER BY disjoin_start + + -- Returns the partition: [0,10), [10,20), [20,30) + +**Split features against a mask:** + +.. code-block:: sql + + SELECT name, disjoin_start, disjoin_end + FROM DISJOIN(features, reference := mask) + +**Re-tile against a uniform grid:** + +.. code-block:: sql + + WITH bins AS ( + SELECT 'chr1' AS chrom, x AS start, x + 1000 AS "end" + FROM range(0, 250000000, 1000) AS t(x) + ) + SELECT * FROM DISJOIN(features, reference := bins) + +.. note:: + + ``range()`` is DuckDB-specific table-generating syntax; on other engines + build the bin grid with the equivalent construct. The grid must span every + chromosome present in ``features`` -- a feature on a chromosome the grid + omits has no covering bin and is dropped by the coverage filter. + +.. note:: + + ``DISJOIN`` is a table function and appears in the ``FROM`` clause, like + ``NEAREST``. On PostgreSQL the derived table must be given an alias + (``FROM DISJOIN(features) AS d``). + +.. note:: + + The appended ``disjoin_start`` / ``disjoin_end`` columns are emitted in the + target table's coordinate system -- the same convention as its + passed-through ``start`` / ``end`` columns, so every column of an output row + shares one convention. Cut positions are computed canonically inside the + operator; only their final representation follows the target table. + +.. note:: + + ``DISJOIN`` operates on coordinates only and is strand-blind: a reference + interval cuts a target interval regardless of either one's strand. To + disjoin per strand, filter the target and reference to a single strand + before the call. + +Related Operators +~~~~~~~~~~~~~~~~~ + +- :ref:`MERGE ` - Collapse overlapping intervals into one +- :ref:`CLUSTER ` - Assign cluster IDs without splitting diff --git a/docs/index.rst b/docs/index.rst index 718f0ab..71237d6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ See the :doc:`GIQL dialect ` docs. dialect/spatial-operators dialect/distance-operators dialect/aggregation-operators + dialect/set-operators dialect/quantifiers Transpilation diff --git a/src/giql/mcp/server.py b/src/giql/mcp/server.py index 1de3565..194abb7 100644 --- a/src/giql/mcp/server.py +++ b/src/giql/mcp/server.py @@ -136,7 +136,7 @@ ], "returns": "Each target row with the sub-interval appended (disjoin_chrom, disjoin_start, disjoin_end)", "example": "SELECT * FROM DISJOIN(features, reference := mask)", - "doc_file": "dialect/aggregation-operators.rst", + "doc_file": "dialect/set-operators.rst", }, "ANY": { "category": "quantifier", @@ -221,6 +221,7 @@ def find_docs_root() -> Path | None: "dialect/spatial-operators": "dialect/spatial-operators.rst", "dialect/distance-operators": "dialect/distance-operators.rst", "dialect/aggregation-operators": "dialect/aggregation-operators.rst", + "dialect/set-operators": "dialect/set-operators.rst", "dialect/quantifiers": "dialect/quantifiers.rst", "transpilation/index": "transpilation/index.rst", "transpilation/api-reference": "transpilation/api-reference.rst", @@ -303,7 +304,7 @@ def get_documentation(path: str) -> str: Available paths: - index, quickstart - dialect/index, dialect/spatial-operators, dialect/distance-operators, - dialect/aggregation-operators, dialect/quantifiers + dialect/aggregation-operators, dialect/set-operators, dialect/quantifiers - transpilation/index, transpilation/api-reference, transpilation/execution, transpilation/performance, transpilation/schema-mapping From a3c61688046ab80da4ef4f3e42d8a1a2cf996e92 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 16:06:01 -0400 Subject: [PATCH 010/142] docs: Sharpen DISJOIN recipe use-case descriptions Ground each recipe in a concrete genomics scenario -- pooled ChIP-seq peak calls, ATAC-seq features against a callable mask, a fixed-width binned coverage matrix -- and note that the uniform-grid recipe relies on DuckDB-specific range() syntax and must span every chromosome present in the input. --- docs/recipes/disjoin.rst | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/docs/recipes/disjoin.rst b/docs/recipes/disjoin.rst index 775c398..1164ce5 100644 --- a/docs/recipes/disjoin.rst +++ b/docs/recipes/disjoin.rst @@ -20,9 +20,10 @@ sub-intervals defined by its own breakpoints: FROM DISJOIN(features) ORDER BY disjoin_chrom, disjoin_start -**Use case:** Produce a non-overlapping segment track -- the equivalent of -Bioconductor's ``disjoin()`` -- that downstream queries can aggregate without -double-counting overlaps. +**Use case:** Given ChIP-seq peak calls pooled from several samples, produce a +non-overlapping segment track -- the equivalent of Bioconductor's ``disjoin()`` +-- so downstream signal or count aggregates never double-count a base covered +by two overlapping sample peaks. Track Each Segment's Parent ~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -52,8 +53,9 @@ the pieces the mask covers: SELECT name, disjoin_start, disjoin_end FROM DISJOIN(features, reference := mask) -**Use case:** Restrict features to mask regions while splitting them at mask -boundaries so no piece straddles a mask edge. +**Use case:** Restrict ATAC-seq or gene features to a set of callable (or +otherwise interesting) mask regions, splitting them at the mask boundaries so +no reported piece straddles a mask edge. Re-tile Against a Uniform Grid ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -68,8 +70,15 @@ Pass a generated set of fixed-width bins as the reference: ) SELECT * FROM DISJOIN(features, reference := bins) -**Use case:** Break features onto a uniform coordinate grid so each piece -falls within a single bin. +**Use case:** Break features onto a uniform coordinate grid -- for example to +build a fixed-width binned coverage matrix -- so each piece falls within a +single bin. + +.. note:: + + ``range()`` is DuckDB-specific syntax for generating the bin grid; other + engines need their own generator. The grid must also span every chromosome + present in ``features``, or features on an uncovered chromosome are dropped. Coming from Bedtools? --------------------- From f1a7667badfc9bb95a0a16d1aa84dbeb271e33d3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 21:04:22 -0400 Subject: [PATCH 011/142] fix: Reject table-function calls that supply no target DISJOIN, NEAREST, MERGE, and CLUSTER each bound the target only when a positional argument was present, so a call with no target -- such as DISJOIN(reference := refs) or MERGE(stranded := true) -- built a node with the target unset. The failure surfaced far downstream as the misleading "Target table 'None' not found". Each from_arg_list now raises a ParseError naming the missing target the moment the call is parsed, so the diagnostic points at the real problem. --- src/giql/expressions.py | 18 ++++++++++++++++++ tests/test_cluster_parsing.py | 20 ++++++++++++++++++++ tests/test_disjoin_parsing.py | 17 +++++++++++++++++ tests/test_merge_parsing.py | 19 +++++++++++++++++++ tests/test_nearest_parsing.py | 17 +++++++++++++++++ 5 files changed, 91 insertions(+) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 747d63b..fd97a62 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -112,6 +112,11 @@ def from_arg_list(cls, args): kwargs["this"] = positional_args[0] if len(positional_args) > 1: kwargs["distance"] = positional_args[1] + if "this" not in kwargs: + raise ParseError( + "CLUSTER requires a genomic interval column as its first " + "argument." + ) return cls(**kwargs) @@ -140,6 +145,11 @@ def from_arg_list(cls, args): kwargs["this"] = positional_args[0] if len(positional_args) > 1: kwargs["distance"] = positional_args[1] + if "this" not in kwargs: + raise ParseError( + "MERGE requires a genomic interval column as its first " + "argument." + ) return cls(**kwargs) @@ -201,6 +211,10 @@ def from_arg_list(cls, args): kwargs, positional_args = _split_named_and_positional(args) if len(positional_args) >= 1: kwargs["this"] = positional_args[0] + if "this" not in kwargs: + raise ParseError( + "NEAREST requires a target table as its first argument." + ) return cls(**kwargs) @@ -243,4 +257,8 @@ def from_arg_list(cls, args): ) if len(positional_args) >= 1: kwargs["this"] = positional_args[0] + if "this" not in kwargs: + raise ParseError( + "DISJOIN requires a target table as its first argument." + ) return cls(**kwargs) diff --git a/tests/test_cluster_parsing.py b/tests/test_cluster_parsing.py index f0f9039..adfb116 100644 --- a/tests/test_cluster_parsing.py +++ b/tests/test_cluster_parsing.py @@ -5,7 +5,9 @@ only := and => are accepted for named parameter binding. """ +import pytest from sqlglot import parse_one +from sqlglot.errors import ParseError from giql.dialect import GIQLDialect from giql.expressions import GIQLCluster @@ -75,3 +77,21 @@ def test_from_arg_list_with_kwarg_syntax(self): assert cluster_expr.args.get("stranded") is not None, ( "Missing stranded parameter with => syntax" ) + + def test_from_arg_list_should_reject_missing_target(self): + """Test that a CLUSTER call with no interval argument is rejected. + + Given: + A GIQL query with CLUSTER(stranded := true) supplying only a + named argument and no positional genomic interval column. + When: + Parsing the query. + Then: + It should raise a ParseError naming the required column argument. + """ + # Arrange, act, & assert + with pytest.raises(ParseError, match="requires a genomic interval"): + parse_one( + "SELECT CLUSTER(stranded := true) AS cluster_id FROM peaks", + dialect=GIQLDialect, + ) diff --git a/tests/test_disjoin_parsing.py b/tests/test_disjoin_parsing.py index a67a68c..3e870c8 100644 --- a/tests/test_disjoin_parsing.py +++ b/tests/test_disjoin_parsing.py @@ -103,6 +103,23 @@ def test_from_arg_list_should_reject_unknown_named_argument(self): "SELECT * FROM DISJOIN(features, mask := refs)", dialect=GIQLDialect ) + def test_from_arg_list_should_reject_missing_target(self): + """Test that a DISJOIN call with no target argument is rejected. + + Given: + A GIQL query with DISJOIN(reference := refs) supplying only a + named reference and no positional target. + When: + Parsing the query. + Then: + It should raise a ParseError naming the required target argument. + """ + # Arrange, act, & assert + with pytest.raises(ParseError, match="requires a target table"): + parse_one( + "SELECT * FROM DISJOIN(reference := refs)", dialect=GIQLDialect + ) + def test_from_arg_list_should_map_reference_when_named_with_walrus(self): """Test that a walrus-named reference argument is carried. diff --git a/tests/test_merge_parsing.py b/tests/test_merge_parsing.py index d97f048..6ae104c 100644 --- a/tests/test_merge_parsing.py +++ b/tests/test_merge_parsing.py @@ -5,7 +5,9 @@ named parameter binding. """ +import pytest from sqlglot import parse_one +from sqlglot.errors import ParseError from giql.dialect import GIQLDialect from giql.expressions import GIQLMerge @@ -68,3 +70,20 @@ def test_from_arg_list_with_kwarg_syntax(self): assert merge_expr.args.get("stranded") is not None, ( "Missing stranded parameter with => syntax" ) + + def test_from_arg_list_should_reject_missing_target(self): + """Test that a MERGE call with no interval argument is rejected. + + Given: + A GIQL query with MERGE(stranded := true) supplying only a named + argument and no positional genomic interval column. + When: + Parsing the query. + Then: + It should raise a ParseError naming the required column argument. + """ + # Arrange, act, & assert + with pytest.raises(ParseError, match="requires a genomic interval"): + parse_one( + "SELECT MERGE(stranded := true) FROM peaks", dialect=GIQLDialect + ) diff --git a/tests/test_nearest_parsing.py b/tests/test_nearest_parsing.py index 9e07035..044d570 100644 --- a/tests/test_nearest_parsing.py +++ b/tests/test_nearest_parsing.py @@ -4,7 +4,9 @@ NEAREST function calls with various argument patterns. """ +import pytest from sqlglot import parse_one +from sqlglot.errors import ParseError from giql.dialect import GIQLDialect from giql.expressions import GIQLNearest @@ -244,3 +246,18 @@ def test_from_arg_list_with_kwarg_syntax(self): assert nearest_expr.args.get("k") is not None, ( "Missing k parameter with => syntax" ) + + def test_from_arg_list_should_reject_missing_target(self): + """Test that a NEAREST call with no target argument is rejected. + + Given: + A GIQL query with NEAREST(k := 3) supplying only named arguments + and no positional target table. + When: + Parsing the query. + Then: + It should raise a ParseError naming the required target argument. + """ + # Arrange, act, & assert + with pytest.raises(ParseError, match="requires a target table"): + parse_one("SELECT * FROM NEAREST(k := 3)", dialect=GIQLDialect) From 9d16e018c9af456e39547315a4796da333819c6a Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 21:04:45 -0400 Subject: [PATCH 012/142] fix: Reject a DISJOIN target using the reserved CTE prefix DISJOIN names its internal CTEs with a __giql_dj_ prefix and already rejects a reference relation that uses it. A target table with the same prefix was unguarded and would emit a self-referential CTE that fails with no actionable error. giqldisjoin_sql now rejects a resolved target name carrying the reserved prefix, symmetric with the existing reference check. --- src/giql/generators/base.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index a742065..ac3681d 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -296,6 +296,15 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: ) target_table = self.tables.get(target_name) + # The __giql_dj_ prefix names the operator's internal CTEs; a target + # table using it would collide with them. + if target_name.startswith("__giql_dj_"): + raise ValueError( + f"DISJOIN target {target_name!r} uses the reserved " + "'__giql_dj_' prefix, which names the operator's internal " + "CTEs. Rename the table." + ) + # Resolve the reference relation (defaults to the target set). ref_from, ref_chrom, ref_start, ref_end, ref_table = ( self._resolve_disjoin_reference( From c8125fa419286aeb5b8b259d4c002ec4640ceb24 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 21:05:03 -0400 Subject: [PATCH 013/142] refactor: Rebuild giqldisjoin_sql from named CTE fragments The DISJOIN SQL was assembled as one ~30-fragment string concatenation, where a dropped inter-fragment space would silently produce invalid SQL. Build each __giql_dj_ CTE as a named local and join them once at the return, so every CTE reads as a discrete block. The generated SQL is unchanged. --- src/giql/generators/base.py | 78 ++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index ac3681d..c965ddf 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -333,42 +333,50 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: out_start = self._decanonical_start("s.seg_start", target_table) out_end = self._decanonical_end("s.seg_end", target_table) - # The `seg_end > seg_start` guard in the final WHERE is belt-and- - # suspenders: UNION already dedupes cut positions, so LEAD cannot - # produce a zero-length segment unless that UNION becomes UNION ALL. + # Build the WITH clause one named fragment per __giql_dj_* CTE so each + # block reads on its own. The `seg_end > seg_start` guard in the final + # WHERE is belt-and-suspenders: UNION already dedupes cut positions, so + # LEAD cannot produce a zero-length segment unless it becomes UNION ALL. + ref_cte = f"__giql_dj_ref AS (SELECT * FROM {ref_from})" + tgt_cte = f"__giql_dj_tgt AS (SELECT * FROM {target_name})" + bp_cte = ( + "__giql_dj_bp AS (" + f'SELECT "{ref_chrom}" AS chrom, {bp_start} AS pos FROM __giql_dj_ref ' + "UNION " + f'SELECT "{ref_chrom}" AS chrom, {bp_end} AS pos FROM __giql_dj_ref)' + ) + cuts_cte = ( + "__giql_dj_cuts AS (" + f'SELECT t."{target_chrom}" AS kc, t."{target_start}" AS ks, ' + f't."{target_end}" AS ke, {t_start} AS pos FROM __giql_dj_tgt AS t ' + "UNION " + f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}", ' + f"{t_end} FROM __giql_dj_tgt AS t " + "UNION " + f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}", ' + "bp.pos FROM __giql_dj_tgt AS t JOIN __giql_dj_bp AS bp " + f"ON bp.chrom = {t_chrom} AND bp.pos > {t_start} " + f"AND bp.pos < {t_end})" + ) + segs_cte = ( + "__giql_dj_segs AS (" + "SELECT kc, ks, ke, pos AS seg_start, " + "LEAD(pos) OVER (PARTITION BY kc, ks, ke ORDER BY pos) AS seg_end " + "FROM __giql_dj_cuts)" + ) + final_select = ( + f"SELECT t.*, s.kc AS disjoin_chrom, {out_start} AS disjoin_start, " + f"{out_end} AS disjoin_end FROM __giql_dj_tgt AS t " + f'JOIN __giql_dj_segs AS s ON t."{target_chrom}" = s.kc ' + f'AND t."{target_start}" = s.ks AND t."{target_end}" = s.ke ' + "WHERE s.seg_end IS NOT NULL AND s.seg_end > s.seg_start " + "AND EXISTS (SELECT 1 FROM __giql_dj_ref AS r " + f'WHERE r."{ref_chrom}" = s.kc AND {r_start} <= s.seg_start ' + f"AND {r_end} > s.seg_start)" + ) return ( - "(WITH __giql_dj_ref AS (" - f"SELECT * FROM {ref_from}" - "), __giql_dj_tgt AS (" - f"SELECT * FROM {target_name}" - "), __giql_dj_bp AS (" - f'SELECT "{ref_chrom}" AS chrom, {bp_start} AS pos FROM __giql_dj_ref' - " UNION " - f'SELECT "{ref_chrom}" AS chrom, {bp_end} AS pos FROM __giql_dj_ref' - "), __giql_dj_cuts AS (" - f'SELECT t."{target_chrom}" AS kc, t."{target_start}" AS ks,' - f' t."{target_end}" AS ke, {t_start} AS pos FROM __giql_dj_tgt AS t' - " UNION " - f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}",' - f" {t_end} FROM __giql_dj_tgt AS t" - " UNION " - f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}",' - " bp.pos FROM __giql_dj_tgt AS t JOIN __giql_dj_bp AS bp" - f" ON bp.chrom = {t_chrom} AND bp.pos > {t_start}" - f" AND bp.pos < {t_end}" - "), __giql_dj_segs AS (" - "SELECT kc, ks, ke, pos AS seg_start," - " LEAD(pos) OVER (PARTITION BY kc, ks, ke ORDER BY pos) AS seg_end" - " FROM __giql_dj_cuts" - ") SELECT t.*, s.kc AS disjoin_chrom," - f" {out_start} AS disjoin_start, {out_end} AS disjoin_end" - " FROM __giql_dj_tgt AS t JOIN __giql_dj_segs AS s" - f' ON t."{target_chrom}" = s.kc AND t."{target_start}" = s.ks' - f' AND t."{target_end}" = s.ke' - " WHERE s.seg_end IS NOT NULL AND s.seg_end > s.seg_start" - " AND EXISTS (SELECT 1 FROM __giql_dj_ref AS r WHERE" - f' r."{ref_chrom}" = s.kc AND {r_start} <= s.seg_start' - f" AND {r_end} > s.seg_start))" + f"(WITH {ref_cte}, {tgt_cte}, {bp_cte}, " + f"{cuts_cte}, {segs_cte} {final_select})" ) def giqldistance_sql(self, expression: GIQLDistance) -> str: From 9717f8c6cf786be0b7c4ef51d8e96199ee274b20 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 21:05:20 -0400 Subject: [PATCH 014/142] test: Restructure DISJOIN test docstrings and cover the target guard The DISJOIN transpilation and execution test docstrings used a free-form GIVEN/WHEN/THEN prose block. Convert them to the structured Given/When/Then form the test guide specifies and add the Arrange-Act-Assert phase comments, matching test_disjoin_parsing.py. Also add a transpilation test for the reserved-prefix target guard. --- tests/test_disjoin_transpilation.py | 199 ++++++++++++++++++++-------- tests/test_disjoin_udf.py | 143 ++++++++++++++------ 2 files changed, 249 insertions(+), 93 deletions(-) diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index eb37e15..2388e3e 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -15,74 +15,110 @@ class TestDisjoinTranspilation: """Tests for DISJOIN transpilation to SQL.""" def test_giqldisjoin_sql_should_emit_with_cte_subquery(self): + """Test that DISJOIN transpiles to a WITH-CTE subquery. + + Given: + A GIQL query selecting from DISJOIN(features). + When: + Transpiling to SQL. + Then: + It should be a WITH-CTE subquery using UNION, LEAD, and EXISTS. """ - GIVEN a GIQL query selecting from DISJOIN(features) - WHEN transpiling to SQL - THEN the output should be a WITH-CTE subquery using UNION, LEAD and EXISTS - """ + # Arrange & act sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]).upper() + # Assert assert "WITH __GIQL_DJ_REF" in sql assert "UNION" in sql assert "LEAD(" in sql assert "EXISTS (" in sql def test_giqldisjoin_sql_should_emit_disjoin_columns(self): + """Test that DISJOIN appends the sub-interval columns. + + Given: + A GIQL query selecting from DISJOIN(features). + When: + Transpiling to SQL. + Then: + It should append the sub-interval as disjoin_chrom, disjoin_start, + and disjoin_end. """ - GIVEN a GIQL query selecting from DISJOIN(features) - WHEN transpiling to SQL - THEN the sub-interval should be appended as disjoin_chrom/start/end - """ + # Arrange & act sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) + # Assert assert "AS disjoin_chrom" in sql assert "AS disjoin_start" in sql assert "AS disjoin_end" in sql def test_giqldisjoin_sql_should_default_reference_to_target_when_omitted(self): + """Test that an omitted reference defaults to the target set. + + Given: + A DISJOIN call with no reference argument. + When: + Transpiling to SQL. + Then: + It should make the reference CTE select from the target table. """ - GIVEN a DISJOIN call with no reference argument - WHEN transpiling to SQL - THEN the reference CTE should select from the target table - """ + # Arrange & act sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) + # Assert assert "__giql_dj_ref AS (SELECT * FROM features)" in sql def test_giqldisjoin_sql_should_use_explicit_reference_relation(self): + """Test that an explicit reference table is used as the reference. + + Given: + A DISJOIN call with an explicit reference table. + When: + Transpiling to SQL. + Then: + It should make the reference CTE select from that reference table. """ - GIVEN a DISJOIN call with an explicit reference table - WHEN transpiling to SQL - THEN the reference CTE should select from that reference table - """ + # Arrange & act sql = transpile( "SELECT * FROM DISJOIN(features, reference := refs)", tables=["features", "refs"], ) + # Assert assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql def test_giqldisjoin_sql_should_honor_custom_column_names(self): + """Test that custom genomic column names are honored. + + Given: + A target table with custom genomic column names. + When: + Transpiling a DISJOIN query. + Then: + It should reference the custom column names in the generated SQL. """ - GIVEN a target table with custom genomic column names - WHEN transpiling a DISJOIN query - THEN the generated SQL should reference the custom column names - """ + # Arrange & act sql = transpile( "SELECT * FROM DISJOIN(feats)", tables=[Table("feats", chrom_col="seqid", start_col="lo", end_col="hi")], ) + # Assert assert 't."seqid"' in sql assert 't."lo"' in sql assert 't."hi"' in sql def test_giqldisjoin_sql_should_canonicalize_one_based_closed_target(self): + """Test that a 1-based closed target is canonicalized. + + Given: + A target table stored as 1-based closed intervals. + When: + Transpiling a DISJOIN query. + Then: + It should shift the start to canonical 0-based coordinates. """ - GIVEN a target table stored as 1-based closed intervals - WHEN transpiling a DISJOIN query - THEN the start should be shifted to canonical 0-based coordinates - """ + # Arrange & act sql = transpile( "SELECT * FROM DISJOIN(features)", tables=[ @@ -94,14 +130,20 @@ def test_giqldisjoin_sql_should_canonicalize_one_based_closed_target(self): ], ) + # Assert assert '(t."start" - 1)' in sql def test_giqldisjoin_sql_should_emit_disjoin_start_in_target_encoding(self): + """Test that disjoin_start is emitted in the target's encoding. + + Given: + A target table stored as 1-based closed intervals. + When: + Transpiling a DISJOIN query. + Then: + It should shift disjoin_start back to the target's encoding. """ - GIVEN a target table stored as 1-based closed intervals - WHEN transpiling a DISJOIN query - THEN disjoin_start should be shifted back to the target's encoding - """ + # Arrange & act sql = transpile( "SELECT * FROM DISJOIN(features)", tables=[ @@ -113,41 +155,60 @@ def test_giqldisjoin_sql_should_emit_disjoin_start_in_target_encoding(self): ], ) + # Assert assert "(s.seg_start + 1) AS disjoin_start" in sql def test_giqldisjoin_sql_should_inline_subquery_reference(self): + """Test that a subquery reference is inlined as a derived table. + + Given: + A DISJOIN call whose reference is a subquery. + When: + Transpiling to SQL. + Then: + It should inline the subquery as an aliased derived table. """ - GIVEN a DISJOIN call whose reference is a subquery - WHEN transpiling to SQL - THEN the subquery should be inlined as an aliased derived table - """ + # Arrange & act sql = transpile( "SELECT * FROM DISJOIN(features, reference := (SELECT * FROM refs))", tables=["features", "refs"], ) + # Assert assert "AS __giql_dj_rs" in sql def test_giqldisjoin_sql_should_resolve_in_query_cte_reference(self): + """Test that a reference naming an in-query CTE resolves to it. + + Given: + A DISJOIN reference naming a CTE defined in the outer query. + When: + Transpiling to SQL. + Then: + It should make the reference CTE select from that CTE name. """ - GIVEN a DISJOIN reference naming a CTE defined in the outer query - WHEN transpiling to SQL - THEN the reference CTE should select from that CTE name - """ + # Arrange & act sql = transpile( "WITH bins AS (SELECT 1) " "SELECT * FROM DISJOIN(features, reference := bins)", tables=["features"], ) + # Assert assert "__giql_dj_ref AS (SELECT * FROM bins)" in sql def test_giqldisjoin_sql_should_let_cte_shadow_registered_table(self): + """Test that an in-query CTE shadows a registered table of the same name. + + Given: + A reference name shared by a registered 1-based table and a CTE. + When: + Transpiling to SQL. + Then: + It should let the CTE shadow the table and leave canonical columns + unshifted. """ - GIVEN a reference name shared by a registered 1-based table and a CTE - WHEN transpiling to SQL - THEN the CTE should shadow the table and canonical columns stay unshifted - """ + # Arrange & act sql = transpile( "WITH refs AS (SELECT 1) " "SELECT * FROM DISJOIN(features, reference := refs)", @@ -157,15 +218,21 @@ def test_giqldisjoin_sql_should_let_cte_shadow_registered_table(self): ], ) + # Assert assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql assert '("start" - 1)' not in sql def test_giqldisjoin_sql_should_reject_literal_range_reference(self): + """Test that a literal range reference is rejected. + + Given: + A DISJOIN call whose reference is a literal genomic range. + When: + Transpiling to SQL. + Then: + It should raise a ValueError rejecting the reference form. """ - GIVEN a DISJOIN call whose reference is a literal genomic range - WHEN transpiling to SQL - THEN transpilation should raise an error rejecting the reference form - """ + # Arrange, act, & assert with pytest.raises(ValueError, match="DISJOIN reference must be"): transpile( "SELECT * FROM DISJOIN(features, reference := 'chr1:1-9')", @@ -173,23 +240,51 @@ def test_giqldisjoin_sql_should_reject_literal_range_reference(self): ) def test_giqldisjoin_sql_should_reject_reserved_prefix_reference(self): + """Test that a reference name using the reserved prefix is rejected. + + Given: + A DISJOIN reference name using the reserved __giql_dj_ prefix. + When: + Transpiling to SQL. + Then: + It should raise a ValueError naming the reserved prefix. """ - GIVEN a DISJOIN reference name using the reserved __giql_dj_ prefix - WHEN transpiling to SQL - THEN transpilation should raise an error naming the reserved prefix - """ + # Arrange, act, & assert with pytest.raises(ValueError, match="reserved"): transpile( "SELECT * FROM DISJOIN(features, reference := __giql_dj_ref)", tables=["features"], ) - def test_giqldisjoin_sql_should_reject_unknown_reference_name(self): + def test_giqldisjoin_sql_should_reject_reserved_prefix_target(self): + """Test that a target name using the reserved prefix is rejected. + + Given: + A DISJOIN target table whose name uses the reserved __giql_dj_ + prefix. + When: + Transpiling to SQL. + Then: + It should raise a ValueError naming the reserved prefix. """ - GIVEN a DISJOIN reference that is neither a registered table nor a CTE - WHEN transpiling to SQL - THEN transpilation should raise an error rejecting the unknown name + # Arrange, act, & assert + with pytest.raises(ValueError, match="reserved"): + transpile( + "SELECT * FROM DISJOIN(__giql_dj_tgt)", + tables=["__giql_dj_tgt"], + ) + + def test_giqldisjoin_sql_should_reject_unknown_reference_name(self): + """Test that a reference matching neither a table nor a CTE is rejected. + + Given: + A DISJOIN reference that is neither a registered table nor a CTE. + When: + Transpiling to SQL. + Then: + It should raise a ValueError rejecting the unknown name. """ + # Arrange, act, & assert with pytest.raises(ValueError, match="neither a registered table"): transpile( "SELECT * FROM DISJOIN(features, reference := missing)", diff --git a/tests/test_disjoin_udf.py b/tests/test_disjoin_udf.py index f55803d..1509780 100644 --- a/tests/test_disjoin_udf.py +++ b/tests/test_disjoin_udf.py @@ -61,11 +61,16 @@ class TestDisjoinExecution: """End-to-end DISJOIN correctness on every supported backend.""" def test_disjoin_should_split_intervals_at_breakpoints(self, run): + """Test that DISJOIN splits intervals at interior breakpoints. + + Given: + Two overlapping target intervals A=[0,20) and B=[10,30). + When: + DISJOIN runs in self-mode. + Then: + It should split each interval at the other's interior breakpoint. """ - GIVEN two overlapping target intervals A=[0,20) and B=[10,30) - WHEN DISJOIN runs in self-mode - THEN each interval should be split at the other's interior breakpoint - """ + # Arrange & act result = run( "SELECT name, disjoin_start, disjoin_end FROM DISJOIN(features) " "ORDER BY name, disjoin_start", @@ -73,6 +78,7 @@ def test_disjoin_should_split_intervals_at_breakpoints(self, run): features=[("chr1", 0, 20, "A"), ("chr1", 10, 30, "B")], ) + # Assert assert result == [ ("A", 0, 10), ("A", 10, 20), @@ -81,11 +87,16 @@ def test_disjoin_should_split_intervals_at_breakpoints(self, run): ] def test_disjoin_should_yield_bioconductor_partition_when_distinct(self, run): + """Test that DISTINCT sub-intervals yield the Bioconductor partition. + + Given: + Overlapping target intervals in self-mode. + When: + Selecting DISTINCT sub-intervals from DISJOIN. + Then: + It should yield the globally non-overlapping partition. """ - GIVEN overlapping target intervals in self-mode - WHEN selecting DISTINCT sub-intervals from DISJOIN - THEN the result should be the globally non-overlapping partition - """ + # Arrange & act result = run( "SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end " "FROM DISJOIN(features) ORDER BY disjoin_start", @@ -93,14 +104,20 @@ def test_disjoin_should_yield_bioconductor_partition_when_distinct(self, run): features=[("chr1", 0, 20, "A"), ("chr1", 10, 30, "B")], ) + # Assert assert result == [("chr1", 0, 10), ("chr1", 10, 20), ("chr1", 20, 30)] def test_disjoin_should_split_against_explicit_reference(self, run): + """Test that DISJOIN cuts the target at an explicit reference. + + Given: + A target interval and a separate reference set. + When: + DISJOIN runs with an explicit reference. + Then: + It should cut the target at the reference breakpoints. """ - GIVEN a target interval and a separate reference set - WHEN DISJOIN runs with an explicit reference - THEN the target should be cut at the reference breakpoints - """ + # Arrange & act result = run( "SELECT disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", @@ -109,14 +126,20 @@ def test_disjoin_should_split_against_explicit_reference(self, run): refs=[("chr1", 0, 10, "a"), ("chr1", 10, 30, "b")], ) + # Assert assert result == [(0, 10), (10, 30)] def test_disjoin_should_drop_pieces_overlapping_no_reference(self, run): + """Test that sub-intervals overlapping no reference are dropped. + + Given: + A target spanning a gap in the reference coverage. + When: + DISJOIN runs with an explicit reference. + Then: + It should drop sub-intervals overlapping no reference interval. """ - GIVEN a target spanning a gap in the reference coverage - WHEN DISJOIN runs with an explicit reference - THEN sub-intervals overlapping no reference interval should be dropped - """ + # Arrange & act result = run( "SELECT disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", @@ -125,28 +148,40 @@ def test_disjoin_should_drop_pieces_overlapping_no_reference(self, run): refs=[("chr1", 0, 10, "a"), ("chr1", 20, 30, "b")], ) + # Assert assert result == [(0, 10), (20, 30)] def test_disjoin_should_yield_no_rows_when_target_is_a_point(self, run): + """Test that a zero-length target produces no rows. + + Given: + A zero-length target interval. + When: + DISJOIN runs. + Then: + It should produce no output rows. """ - GIVEN a zero-length target interval - WHEN DISJOIN runs - THEN it should produce no output rows - """ + # Arrange & act result = run( "SELECT * FROM DISJOIN(features)", tables=["features"], features=[("chr1", 5, 5, "P")], ) + # Assert assert result == [] def test_disjoin_should_split_duplicate_targets_independently(self, run): + """Test that duplicate target rows are split independently. + + Given: + Two target rows with identical geometry. + When: + DISJOIN runs with a reference that cuts them. + Then: + It should split each duplicate row independently. """ - GIVEN two target rows with identical geometry - WHEN DISJOIN runs with a reference that cuts them - THEN each duplicate row should be split independently - """ + # Arrange & act result = run( "SELECT name, disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs) ORDER BY name, disjoin_start", @@ -155,14 +190,21 @@ def test_disjoin_should_split_duplicate_targets_independently(self, run): refs=[("chr1", 0, 5, "a"), ("chr1", 5, 10, "b")], ) + # Assert assert result == [("X", 0, 5), ("X", 5, 10), ("Y", 0, 5), ("Y", 5, 10)] def test_disjoin_should_not_cross_chromosome_boundaries(self, run): + """Test that sub-intervals never cross a chromosome boundary. + + Given: + Target intervals on different chromosomes. + When: + DISJOIN runs in self-mode. + Then: + It should never produce a sub-interval spanning a chromosome + boundary. """ - GIVEN target intervals on different chromosomes - WHEN DISJOIN runs in self-mode - THEN sub-intervals should never span a chromosome boundary - """ + # Arrange & act result = run( "SELECT disjoin_chrom, disjoin_start, disjoin_end FROM DISJOIN(features) " "ORDER BY disjoin_chrom, disjoin_start", @@ -170,15 +212,21 @@ def test_disjoin_should_not_cross_chromosome_boundaries(self, run): features=[("chr1", 0, 20, "A"), ("chr2", 5, 25, "B")], ) + # Assert assert result == [("chr1", 0, 20), ("chr2", 5, 25)] def test_disjoin_should_pass_through_non_interval_columns(self, run): + """Test that non-interval columns pass through unchanged. + + Given: + A target table carrying a non-interval column. + When: + DISJOIN runs. + Then: + It should carry the intact parent row on every output row + alongside its sub-interval. """ - GIVEN a target table carrying a non-interval column - WHEN DISJOIN runs - THEN every output row should carry the intact parent row alongside - its sub-interval - """ + # Arrange & act result = run( 'SELECT name, chrom, "start", "end", disjoin_start, disjoin_end ' "FROM DISJOIN(features) ORDER BY disjoin_start, name", @@ -186,6 +234,7 @@ def test_disjoin_should_pass_through_non_interval_columns(self, run): features=[("chr1", 0, 20, "A"), ("chr1", 10, 30, "B")], ) + # Assert assert result == [ ("A", "chr1", 0, 20, 0, 10), ("A", "chr1", 0, 20, 10, 20), @@ -194,11 +243,16 @@ def test_disjoin_should_pass_through_non_interval_columns(self, run): ] def test_disjoin_should_accept_a_cte_as_reference(self, run): + """Test that a CTE can be used as the reference. + + Given: + A reference supplied as a CTE defined in the outer query. + When: + DISJOIN runs against that CTE. + Then: + It should split the target at the CTE's breakpoints. """ - GIVEN a reference supplied as a CTE defined in the outer query - WHEN DISJOIN runs against that CTE - THEN the target should be split at the CTE's breakpoints - """ + # Arrange & act result = run( "WITH bins AS (" " SELECT 'chr1' AS chrom, 0 AS \"start\", 10 AS \"end\" " @@ -210,14 +264,20 @@ def test_disjoin_should_accept_a_cte_as_reference(self, run): features=[("chr1", 0, 20, "T")], ) + # Assert assert result == [(0, 10), (10, 20)] def test_disjoin_should_not_cut_at_breakpoint_on_target_boundary(self, run): + """Test that a breakpoint on a target boundary does not split it. + + Given: + A reference breakpoint coinciding with a target boundary. + When: + DISJOIN runs. + Then: + It should not split the target at that boundary. """ - GIVEN a reference breakpoint coinciding with a target boundary - WHEN DISJOIN runs - THEN the target should not be split at that boundary - """ + # Arrange & act result = run( "SELECT disjoin_start, disjoin_end " "FROM DISJOIN(features, reference := refs)", @@ -226,4 +286,5 @@ def test_disjoin_should_not_cut_at_breakpoint_on_target_boundary(self, run): refs=[("chr1", 10, 20, "r")], ) + # Assert assert result == [(10, 20)] From 2f71af46543017841771b0f242a05a3d1681e743 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 21:05:34 -0400 Subject: [PATCH 015/142] test: Tidy the usage-pattern test scaffolding Drop the structural banner-comment rule lines that divide usage_patterns.py into sections, keeping the explanatory prose. Make _annotations delegate to _features_like rather than duplicate it. Hoist the duckdb import in test_usage_patterns.py to module scope. --- tests/test_usage_patterns.py | 3 +-- tests/usage_patterns.py | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/test_usage_patterns.py b/tests/test_usage_patterns.py index 545aad7..c82dca7 100644 --- a/tests/test_usage_patterns.py +++ b/tests/test_usage_patterns.py @@ -17,6 +17,7 @@ :class:`TestDisjoinProfiles`. """ +import duckdb import pytest from giql import transpile @@ -60,8 +61,6 @@ def _run_duckdb(fixtures: tuple[TableFixture, ...], sql: str) -> list: TABLE`, so a fixture carrying a custom `Table` config is materialized with its own physical column names and types. """ - import duckdb - conn = duckdb.connect(":memory:") try: for fixture in fixtures: diff --git a/tests/usage_patterns.py b/tests/usage_patterns.py index 2b7f8c5..bee828f 100644 --- a/tests/usage_patterns.py +++ b/tests/usage_patterns.py @@ -592,7 +592,6 @@ def rendered_cases(usage: OperatorUsage) -> list: return cases -# --------------------------------------------------------------------------- # Canonical fixture tables. # # The intervals are chosen so each pattern produces a non-trivial, @@ -601,7 +600,6 @@ def rendered_cases(usage: OperatorUsage) -> list: # transform some rows. The CANONICAL profile reuses these unchanged so its # snapshot rows stay semantically identical to the originally committed # manifests. -# --------------------------------------------------------------------------- _FEATURES = TableFixture( "features", (("chr1", 0, 100, "a"), ("chr1", 60, 180, "b"), ("chr2", 10, 40, "c")), @@ -626,7 +624,7 @@ def _features_like(name: str, rows: tuple[tuple, ...]) -> TableFixture: # JOINED, SELF_JOIN_OVERLAP, and set-operation patterns. def _annotations(rows: tuple[tuple, ...] = _ANNOTATIONS.rows) -> TableFixture: """Build an ``annotations`` fixture (canonical layout) with the given rows.""" - return TableFixture("annotations", rows) + return _features_like("annotations", rows) def _table_function_profile( @@ -669,14 +667,12 @@ def _table_function_profile( ) -# --------------------------------------------------------------------------- # DISJOIN semantic profiles. # # Each profile is a TableFunctionUsage exercising one DISJOIN edge case. The # functional suite parametrises every usage pattern over every profile and # snapshots one manifest per (profile, mode). CANONICAL reuses the original # self / reference fixtures so its rows match the first committed manifests. -# --------------------------------------------------------------------------- # CANONICAL -- the original benign data profile, self-mode and reference-mode. DISJOIN_USAGE = TableFunctionUsage( From 367aca4e7c316b08b6ba395c78012965d4b3ae2b Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 21:05:40 -0400 Subject: [PATCH 016/142] docs: Refine the DISJOIN reference and recipe pages Note in the set-operators reference that disjoin_chrom, disjoin_start, and disjoin_end are reserved output column names that collide with a same-named target column. Trim the recipe's "Coming from Bedtools?" section to drop analogies it does not deliver, and correct two section underlines to their title lengths. --- docs/dialect/set-operators.rst | 9 +++++++++ docs/recipes/disjoin.rst | 12 +++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/dialect/set-operators.rst b/docs/dialect/set-operators.rst index abf38ff..17787c6 100644 --- a/docs/dialect/set-operators.rst +++ b/docs/dialect/set-operators.rst @@ -88,6 +88,15 @@ nothing is dropped. In reference mode a target interval that overlaps no reference interval at all yields no output rows -- ``DISJOIN`` can drop a target entirely, not merely trim it. +.. note:: + + ``disjoin_chrom`` / ``disjoin_start`` / ``disjoin_end`` are reserved output + column names. If the target table already carries a column with one of + those names, the output will contain two columns of that name; DuckDB + silently renames the second, so ``SELECT disjoin_start`` then resolves to + the passed-through parent column rather than the computed sub-interval. + Rename the conflicting target column before the call. + Examples ~~~~~~~~ diff --git a/docs/recipes/disjoin.rst b/docs/recipes/disjoin.rst index 1164ce5..f6a4d7b 100644 --- a/docs/recipes/disjoin.rst +++ b/docs/recipes/disjoin.rst @@ -6,10 +6,10 @@ GIQL's ``DISJOIN`` operator -- partitioning a set into non-overlapping segments and re-tiling features against a reference grid. Partition a Set of Intervals ------------------------------ +---------------------------- Build a Disjoint Partition -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~ Split a set of intervals into the maximal set of non-overlapping sub-intervals defined by its own breakpoints: @@ -83,8 +83,6 @@ single bin. Coming from Bedtools? --------------------- -``DISJOIN`` has no single bedtools equivalent. The self-mode partition is -closest to the breakpoints implied by ``bedtools merge`` re-split at every -input boundary; splitting against a reference is closest to ``bedtools -intersect`` combined with the reference's boundaries. See the -:doc:`bedtools-migration` guide for related operations. +``DISJOIN`` has no direct bedtools equivalent -- no single bedtools command +splits intervals at breakpoints. See the :doc:`bedtools-migration` guide for +the GIQL operators that do map onto bedtools commands. From 5a9ca697af969f102ffc99262d53b36de474263e Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 18 May 2026 22:03:52 -0400 Subject: [PATCH 017/142] build: Install pytest-manifest into the pixi test environment The usage-pattern snapshot suite depends on pytest-manifest, declared in the dev dependency group but never mirrored into the pixi environment that CI builds. CI therefore ran without the plugin and every snapshot test errored with "fixture 'manifest' not found". pytest-manifest has no conda-forge package, so add it under tool.pixi.pypi-dependencies -- the pixi environment now installs it from PyPI alongside the conda-managed test dependencies. --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index ade33e7..cc61041 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,3 +94,8 @@ pandas = ">=2.0.0" sqlglot = ">=20.0.0,<30" pip = "*" hypothesis = ">=6.148.2,<7" + +[tool.pixi.pypi-dependencies] +# pytest-manifest has no conda-forge package; the test suite's snapshot +# fixtures need it, so install it from PyPI into the pixi environment. +pytest-manifest = ">=1.0" From f36557be41d0fbb437e774aa6f5ca0bfb9bf1dc6 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 19 May 2026 11:23:15 -0400 Subject: [PATCH 018/142] docs: Add DISJOIN examples to the operators demo notebook Add a DISJOIN section to the demo notebook covering self-mode and reference-mode. Self-mode is shown twice: once partitioning the features_a peak set at scale, and once on a small hand-built set of overlapping intervals where every cut is traceable by eye. The reference-mode example splits features_a at the breakpoints of features_b. The summary now lists DISJOIN as the eighth operator. --- demo.ipynb | 149 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 148 insertions(+), 1 deletion(-) diff --git a/demo.ipynb b/demo.ipynb index 71729f3..1d136ae 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -3579,6 +3579,152 @@ "result.head(15)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "---\n", + "\n", + "## 8. DISJOIN operator\n", + "\n", + "**Split intervals at breakpoints** so that no resulting sub-interval ever partially overlaps a reference interval.\n", + "\n", + "Unlike MERGE and CLUSTER, which *collapse* intervals, DISJOIN *multiplies* rows: one input interval becomes one or more sub-interval rows. It is a table-valued function, so it appears in the `FROM` clause like NEAREST. The full input row passes through unchanged and the sub-interval is appended as `disjoin_chrom`, `disjoin_start`, and `disjoin_end`.\n", + "\n", + "In **self-mode**, `DISJOIN(features)` splits a set against its own breakpoints. Selecting the distinct sub-intervals yields the globally non-overlapping partition — the equivalent of Bioconductor's `disjoin()`.\n", + "\n", + "**GIQL Query**: `SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end FROM DISJOIN(features_a)`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giql_query = \"\"\"\n", + " SELECT DISTINCT disjoin_chrom, disjoin_start, disjoin_end\n", + " FROM DISJOIN(features_a)\n", + " ORDER BY disjoin_chrom, disjoin_start\n", + "\"\"\"\n", + "\n", + "sql = transpile(giql_query, tables=[\"features_a\"])\n", + "print(\"Transpiled SQL:\")\n", + "print(sqlglot.transpile(sql, pretty=True)[0])\n", + "print(\"\\n\" + \"=\" * 80 + \"\\n\")\n", + "\n", + "n_original = conn.execute(\"SELECT COUNT(*) FROM features_a\").fetchone()[0]\n", + "result = conn.execute(sql).fetchdf()\n", + "print(\n", + " f\"Result: {len(result)} non-overlapping segments \"\n", + " f\"(from {n_original} original peaks)\"\n", + ")\n", + "result.head(10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Disjoining a set against itself\n", + "\n", + "The example above runs self-mode at scale; on a small, deliberately overlapping set every cut is visible. With no `reference`, `DISJOIN` defaults the reference to the target set, so each interval is split at every *other* interval's interior breakpoint: `A = [0, 100)` is cut where `B` starts (50); `B = [50, 150)` is cut at `A`'s end (100) and `C`'s start (120); `D = [0, 80)` is cut where `E` starts (40). Selecting `DISTINCT disjoin_*` collapses the pieces into the non-overlapping partition.\n", + "\n", + "**GIQL Query**: `SELECT * FROM DISJOIN(toy_peaks)`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# A small set of deliberately overlapping intervals, so the split is visible.\n", + "conn.execute(\"DROP TABLE IF EXISTS toy_peaks\")\n", + "conn.execute(\n", + " 'CREATE TABLE toy_peaks(chrom VARCHAR, \"start\" INTEGER, \"end\" INTEGER, name VARCHAR)'\n", + ")\n", + "conn.executemany(\n", + " \"INSERT INTO toy_peaks VALUES (?, ?, ?, ?)\",\n", + " [\n", + " (\"chr1\", 0, 100, \"A\"),\n", + " (\"chr1\", 50, 150, \"B\"),\n", + " (\"chr1\", 120, 200, \"C\"),\n", + " (\"chr2\", 0, 80, \"D\"),\n", + " (\"chr2\", 40, 80, \"E\"),\n", + " ],\n", + ")\n", + "print(\"Input intervals (overlapping):\")\n", + "print(\n", + " conn.execute(\n", + " 'SELECT name, chrom, start, \"end\" FROM toy_peaks ORDER BY chrom, start'\n", + " ).fetchdf()\n", + ")\n", + "print(\"\\n\" + \"=\" * 80 + \"\\n\")\n", + "\n", + "# With no reference, DISJOIN splits toy_peaks against its own breakpoints.\n", + "giql_query = \"\"\"\n", + " SELECT name, disjoin_chrom, disjoin_start, disjoin_end\n", + " FROM DISJOIN(toy_peaks)\n", + " ORDER BY disjoin_chrom, disjoin_start, name\n", + "\"\"\"\n", + "\n", + "sql = transpile(giql_query, tables=[\"toy_peaks\"])\n", + "print(\"Transpiled SQL:\")\n", + "print(sqlglot.transpile(sql, pretty=True)[0])\n", + "print(\"\\n\" + \"=\" * 80 + \"\\n\")\n", + "\n", + "result = conn.execute(sql).fetchdf()\n", + "print(\n", + " f\"Result: {len(result)} sub-intervals -- each interval split at its \"\n", + " f\"neighbours' breakpoints\"\n", + ")\n", + "result" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Splitting against a reference set\n", + "\n", + "Pass a second interval set as `reference :=` to cut each target feature at that set's breakpoints, keeping only the pieces a reference interval covers. Here every peak in `features_a` is split at the breakpoints of `features_b`; the parent peak coordinates ride along with each sub-interval, and a peak that overlaps no feature in B drops out entirely.\n", + "\n", + "**GIQL Query**: `SELECT * FROM DISJOIN(features_a, reference := features_b)`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "giql_query = \"\"\"\n", + " SELECT\n", + " chrom,\n", + " start AS peak_start,\n", + " \"end\" AS peak_end,\n", + " signal_value,\n", + " disjoin_start,\n", + " disjoin_end\n", + " FROM DISJOIN(features_a, reference := features_b)\n", + " ORDER BY chrom, disjoin_start\n", + "\"\"\"\n", + "\n", + "sql = transpile(giql_query, tables=[\"features_a\", \"features_b\"])\n", + "print(\"Transpiled SQL:\")\n", + "print(sqlglot.transpile(sql, pretty=True)[0])\n", + "print(\"\\n\" + \"=\" * 80 + \"\\n\")\n", + "\n", + "n_original = conn.execute(\"SELECT COUNT(*) FROM features_a\").fetchone()[0]\n", + "result = conn.execute(sql).fetchdf()\n", + "print(\n", + " f\"Result: {len(result)} sub-intervals of A split at B's breakpoints \"\n", + " f\"(from {n_original} original peaks)\"\n", + ")\n", + "result.head(10)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -3587,7 +3733,7 @@ "\n", "## Summary\n", "\n", - "This demo showcased all 7 GIQL operators:\n", + "This demo showcased all 8 GIQL operators:\n", "\n", "1. **INTERSECTS**: Binary predicate for overlapping intervals\n", "2. **WITHIN**: Binary predicate for containment (A within B)\n", @@ -3596,6 +3742,7 @@ "5. **CLUSTER**: Aggregation operator to assign cluster IDs to overlapping intervals\n", "6. **DISTANCE**: UDF operator to calculate genomic distances between intervals\n", "7. **NEAREST**: Table-valued function for finding k-nearest genomic features\n", + "8. **DISJOIN**: Set operator to split intervals at reference breakpoints\n", "\n", "Each operator was:\n", "- Written in GIQL syntax\n", From abc58e98cec8a7128e84bb33b295752db5a4a287 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 20 May 2026 11:59:37 -0400 Subject: [PATCH 019/142] perf: Skip redundant EXISTS clause in self-referencing DISJOIN In self-reference mode -- DISJOIN(features) with the reference omitted, or DISJOIN(features, reference := features) where features is the same registered table -- the outer-WHERE EXISTS coverage subquery is provably always true. Every emitted segment lies inside its parent target row, and that row is itself a member of the reference set, so the semi-join filters nothing but still plans as a full equi+range hash join. Detect self-reference at the resolver and propagate a flag into the SQL builder. When the flag is set, omit the AND EXISTS clause; the other CTEs and the seg_end guards are untouched. Subquery references and CTE-shadowed names stay conservative -- proving relation equivalence for those is out of scope -- so EXISTS is retained. Benchmarks in the original issue measure 1.30x speedup at 10K intervals rising to 1.83x at 250K, with byte-identical row sets at every input size. --- src/giql/generators/base.py | 59 +++++++++----- tests/test_disjoin_transpilation.py | 114 ++++++++++++++++++++++++++-- 2 files changed, 147 insertions(+), 26 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index c965ddf..243e613 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -306,7 +306,7 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: ) # Resolve the reference relation (defaults to the target set). - ref_from, ref_chrom, ref_start, ref_end, ref_table = ( + ref_from, ref_chrom, ref_start, ref_end, ref_table, is_self_reference = ( self._resolve_disjoin_reference( expression, target_name, @@ -364,15 +364,26 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: "LEAD(pos) OVER (PARTITION BY kc, ks, ke ORDER BY pos) AS seg_end " "FROM __giql_dj_cuts)" ) + # In self-reference mode the coverage EXISTS is provably always true: + # every emitted segment lies inside its parent target row, and that + # row is itself a member of the reference set. Skip the clause so the + # planner does not waste work on a no-op semi-join. + exists_clause = ( + "" + if is_self_reference + else ( + " AND EXISTS (SELECT 1 FROM __giql_dj_ref AS r " + f'WHERE r."{ref_chrom}" = s.kc AND {r_start} <= s.seg_start ' + f"AND {r_end} > s.seg_start)" + ) + ) final_select = ( f"SELECT t.*, s.kc AS disjoin_chrom, {out_start} AS disjoin_start, " f"{out_end} AS disjoin_end FROM __giql_dj_tgt AS t " f'JOIN __giql_dj_segs AS s ON t."{target_chrom}" = s.kc ' f'AND t."{target_start}" = s.ks AND t."{target_end}" = s.ke ' - "WHERE s.seg_end IS NOT NULL AND s.seg_end > s.seg_start " - "AND EXISTS (SELECT 1 FROM __giql_dj_ref AS r " - f'WHERE r."{ref_chrom}" = s.kc AND {r_start} <= s.seg_start ' - f"AND {r_end} > s.seg_start)" + "WHERE s.seg_end IS NOT NULL AND s.seg_end > s.seg_start" + f"{exists_clause}" ) return ( f"(WITH {ref_cte}, {tgt_cte}, {bp_cte}, " @@ -739,9 +750,7 @@ def _detect_nearest_mode( # (validation will catch missing reference errors later) return "correlated" - def _find_outer_table_in_lateral_join( - self, expression: GIQLNearest - ) -> str | None: + def _find_outer_table_in_lateral_join(self, expression: GIQLNearest) -> str | None: """Find the outer table name in a LATERAL join context. Walks up the AST to find the JOIN clause and extracts the outer table @@ -911,7 +920,7 @@ def _resolve_disjoin_reference( target_name: str, target_cols: tuple[str, str, str], target_table: Table | None, - ) -> tuple[str, str, str, str, Table | None]: + ) -> tuple[str, str, str, str, Table | None, bool]: """Resolve the reference relation for a DISJOIN query. The reference contributes the breakpoints at which target intervals are @@ -933,11 +942,16 @@ def _resolve_disjoin_reference( :param target_table: Table config of the target (the default reference config) :return: - Tuple of ``(from_clause, chrom_col, start_col, end_col, table)`` - where ``from_clause`` is the text following ``FROM`` inside the - ``__giql_dj_ref`` CTE. A subquery or CTE reference is assumed to - expose canonical 0-based half-open ``chrom`` / ``start`` / ``end`` - columns, so ``table`` is ``None`` for those. + Tuple of ``(from_clause, chrom_col, start_col, end_col, table, + is_self_reference)`` where ``from_clause`` is the text following + ``FROM`` inside the ``__giql_dj_ref`` CTE. A subquery or CTE + reference is assumed to expose canonical 0-based half-open + ``chrom`` / ``start`` / ``end`` columns, so ``table`` is ``None`` + for those. ``is_self_reference`` is ``True`` only when the + reference provably resolves to the same registered relation as the + target (omitted reference, or a bare name equal to the target name + that is not shadowed by an enclosing CTE and maps to the same + registered table). :raises ValueError: If the reference is not a table name, a CTE, or a subquery, or if a bare name resolves to neither a registered table nor a CTE. @@ -947,9 +961,11 @@ def _resolve_disjoin_reference( # No reference: default to the target set (single-mode). if reference is None: - return target_name, tc, ts, te, target_table + return target_name, tc, ts, te, target_table, True - # Subquery reference: inline it as an aliased derived table. + # Subquery reference: inline it as an aliased derived table. The + # subquery may textually duplicate the target, but proving equivalence + # is out of scope, so treat it as a distinct relation. if isinstance(reference, (exp.Subquery, exp.Select, exp.Union)): from_clause = f"{self.sql(reference)} AS __giql_dj_rs" return ( @@ -958,6 +974,7 @@ def _resolve_disjoin_reference( DEFAULT_START_COL, DEFAULT_END_COL, None, + False, ) # Anything that is not a bare table/CTE name is unsupported. @@ -979,7 +996,9 @@ def _resolve_disjoin_reference( ) # A CTE from an enclosing WITH shadows a registered table of the same - # name; it is assumed to expose canonical default columns. + # name; it is assumed to expose canonical default columns. A CTE may + # contain rows distinct from any same-named registered table, so it is + # never treated as a self-reference. if ref_name in self._enclosing_cte_names(expression): return ( ref_name, @@ -987,6 +1006,7 @@ def _resolve_disjoin_reference( DEFAULT_START_COL, DEFAULT_END_COL, None, + False, ) ref_table = self.tables.get(ref_name) @@ -997,6 +1017,7 @@ def _resolve_disjoin_reference( ref_table.start_col, ref_table.end_col, ref_table, + ref_name == target_name and ref_table is target_table, ) raise ValueError( f"DISJOIN reference {ref_name!r} is neither a registered table " @@ -1168,9 +1189,7 @@ def _canonical_endpoints( BaseGIQLGenerator._canonical_end(raw_end, table), ) - def _resolve_table_name( - self, column_ref: str, table_name: str | None - ) -> str | None: + def _resolve_table_name(self, column_ref: str, table_name: str | None) -> str | None: """Resolve the underlying table name for a column reference. Precedence: explicit ``table_name`` (caller-provided context) > alias diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index 2388e3e..71dfbf5 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -22,7 +22,7 @@ def test_giqldisjoin_sql_should_emit_with_cte_subquery(self): When: Transpiling to SQL. Then: - It should be a WITH-CTE subquery using UNION, LEAD, and EXISTS. + It should be a WITH-CTE subquery using UNION and LEAD. """ # Arrange & act sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]).upper() @@ -31,7 +31,6 @@ def test_giqldisjoin_sql_should_emit_with_cte_subquery(self): assert "WITH __GIQL_DJ_REF" in sql assert "UNION" in sql assert "LEAD(" in sql - assert "EXISTS (" in sql def test_giqldisjoin_sql_should_emit_disjoin_columns(self): """Test that DISJOIN appends the sub-interval columns. @@ -189,8 +188,7 @@ def test_giqldisjoin_sql_should_resolve_in_query_cte_reference(self): """ # Arrange & act sql = transpile( - "WITH bins AS (SELECT 1) " - "SELECT * FROM DISJOIN(features, reference := bins)", + "WITH bins AS (SELECT 1) SELECT * FROM DISJOIN(features, reference := bins)", tables=["features"], ) @@ -210,8 +208,7 @@ def test_giqldisjoin_sql_should_let_cte_shadow_registered_table(self): """ # Arrange & act sql = transpile( - "WITH refs AS (SELECT 1) " - "SELECT * FROM DISJOIN(features, reference := refs)", + "WITH refs AS (SELECT 1) SELECT * FROM DISJOIN(features, reference := refs)", tables=[ "features", Table("refs", coordinate_system="1based", interval_type="closed"), @@ -290,3 +287,108 @@ def test_giqldisjoin_sql_should_reject_unknown_reference_name(self): "SELECT * FROM DISJOIN(features, reference := missing)", tables=["features"], ) + + def test_giqldisjoin_sql_should_skip_exists_clause_when_reference_omitted(self): + """Test that an omitted reference suppresses the coverage EXISTS clause. + + Given: + A DISJOIN call with no reference argument (self-mode). + When: + Transpiling to SQL. + Then: + It should omit the coverage EXISTS subquery, which is provably + always true when the reference equals the target set. + """ + # Arrange & act + sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) + + # Assert + assert "EXISTS (" not in sql + + def test_giqldisjoin_sql_should_skip_exists_clause_when_reference_resolves_to_target( + self, + ): + """Test that a reference naming the target table suppresses EXISTS. + + Given: + A DISJOIN call whose explicit reference is the target table name. + When: + Transpiling to SQL. + Then: + It should omit the coverage EXISTS subquery, since the reference + resolves to the same registered relation as the target. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features, reference := features)", + tables=["features"], + ) + + # Assert + assert "EXISTS (" not in sql + + def test_giqldisjoin_sql_should_emit_exists_clause_when_reference_is_different_table( + self, + ): + """Test that an explicit distinct reference keeps the EXISTS clause. + + Given: + A DISJOIN call with an explicit reference table distinct from the + target. + When: + Transpiling to SQL. + Then: + It should emit the coverage EXISTS subquery against the reference + CTE. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features, reference := refs)", + tables=["features", "refs"], + ) + + # Assert + assert "EXISTS (" in sql + + def test_giqldisjoin_sql_should_emit_exists_clause_when_cte_shadows_target_name( + self, + ): + """Test that a CTE shadowing the target name keeps the EXISTS clause. + + Given: + A reference name shared by the target table and an in-query CTE. + When: + Transpiling to SQL. + Then: + It should emit the coverage EXISTS subquery, since the CTE may + contain rows distinct from the registered target table. + """ + # Arrange & act + sql = transpile( + "WITH features AS (SELECT 1) " + "SELECT * FROM DISJOIN(features, reference := features)", + tables=["features"], + ) + + # Assert + assert "EXISTS (" in sql + + def test_giqldisjoin_sql_should_emit_exists_clause_when_reference_is_subquery(self): + """Test that a subquery reference keeps the EXISTS clause. + + Given: + A DISJOIN call whose reference is a subquery. + When: + Transpiling to SQL. + Then: + It should emit the coverage EXISTS subquery, since a subquery is + conservatively treated as a relation distinct from the target. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features, reference := (SELECT * FROM features))", + tables=["features"], + ) + + # Assert + assert "EXISTS (" in sql From c534458df083ef782dca89e478a1a8c419d5fead Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 20 May 2026 12:00:47 -0400 Subject: [PATCH 020/142] test: Add edge-case transpilation tests for DISJOIN EXISTS-skip Cover behaviors orthogonal to the test-first cases: composition with non-default coordinate systems (1-based closed) on both self-mode and explicit-self-reference paths; composition with custom genomic column names; per-call independence inside a query that mixes self-mode and distinct-reference DISJOIN calls; and CTE shadowing detected via the ancestor walk past a nested subquery scope. These tests pin the optimization's contract against future regressions in adjacent code paths -- canonicalization, column resolution, the enclosing-CTE search, and per-expression flag computation -- where breakage would not surface in the basic truth-table tests. --- tests/test_disjoin_transpilation.py | 140 ++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index 71dfbf5..bfee25a 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -392,3 +392,143 @@ def test_giqldisjoin_sql_should_emit_exists_clause_when_reference_is_subquery(se # Assert assert "EXISTS (" in sql + + def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_one_based_closed_encoding( + self, + ): + """Test that self-mode skips EXISTS even with non-default coordinates. + + Given: + A self-mode DISJOIN against a target registered as 1-based closed. + When: + Transpiling to SQL. + Then: + It should omit the coverage EXISTS subquery and still emit the + canonical 0-based shift on the target endpoints. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features)", + tables=[ + Table( + "features", + coordinate_system="1based", + interval_type="closed", + ) + ], + ) + + # Assert + assert "EXISTS (" not in sql + assert '(t."start" - 1)' in sql + + def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_custom_column_names( + self, + ): + """Test that self-mode skips EXISTS cleanly under custom column names. + + Given: + A self-mode DISJOIN against a target with custom genomic column + names. + When: + Transpiling to SQL. + Then: + It should omit the coverage EXISTS subquery and leave no dangling + reference-side qualifier (no ``r."seqid"`` / ``r."lo"`` / + ``r."hi"``). + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(feats)", + tables=[Table("feats", chrom_col="seqid", start_col="lo", end_col="hi")], + ) + + # Assert + assert "EXISTS (" not in sql + assert 'r."seqid"' not in sql + assert 'r."lo"' not in sql + assert 'r."hi"' not in sql + + def test_giqldisjoin_sql_should_skip_exists_clause_when_explicit_self_reference_uses_one_based_closed_encoding( + self, + ): + """Test that explicit self-reference skips EXISTS under non-default config. + + Given: + A DISJOIN call with an explicit ``reference := features`` where + ``features`` is registered as 1-based closed. + When: + Transpiling to SQL. + Then: + It should omit the coverage EXISTS subquery and still emit the + canonical 0-based shift on the breakpoint CTE endpoints. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features, reference := features)", + tables=[ + Table( + "features", + coordinate_system="1based", + interval_type="closed", + ) + ], + ) + + # Assert + assert "EXISTS (" not in sql + assert '("start" - 1)' in sql + + def test_giqldisjoin_sql_should_emit_one_exists_clause_when_query_mixes_self_and_distinct_reference_disjoins( + self, + ): + """Test that the EXISTS skip is decided per DISJOIN call. + + Given: + A query containing one self-mode DISJOIN and one distinct-reference + DISJOIN in the same SELECT. + When: + Transpiling to SQL. + Then: + It should produce SQL containing exactly one ``EXISTS (`` + occurrence — only the distinct-reference call retains the + coverage filter. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features) " + "UNION ALL " + "SELECT * FROM DISJOIN(features, reference := refs)", + tables=["features", "refs"], + ) + + # Assert + assert sql.count("EXISTS (") == 1 + + def test_giqldisjoin_sql_should_emit_exists_clause_when_enclosing_cte_shadows_target_from_outer_scope( + self, + ): + """Test that a CTE shadowing the target name from an outer scope keeps EXISTS. + + Given: + A nested query where the outer WITH defines a CTE named ``features`` + and an inner subquery contains + ``DISJOIN(features, reference := features)``, with ``features`` + also a registered table. + When: + Transpiling to SQL. + Then: + It should emit the coverage EXISTS subquery — the enclosing-CTE + check follows the ancestor chain past the immediate WITH. + """ + # Arrange & act + sql = transpile( + "WITH features AS (SELECT 1) " + "SELECT * FROM (" + "SELECT * FROM DISJOIN(features, reference := features)" + ") AS sub", + tables=["features"], + ) + + # Assert + assert "EXISTS (" in sql From 6c0470fdcb436eee6208104e6021b712e9dc0c30 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 20 May 2026 12:01:04 -0400 Subject: [PATCH 021/142] test: Add integration tests validating DISJOIN EXISTS-skip semantics End-to-end coverage through DuckDB for the optimization. The keystone test asserts row-set equivalence between DISJOIN(features) and DISJOIN(features, reference := features) across all four coordinate conventions -- proving the two self-reference paths take the same optimized branch at runtime. The remaining tests guard the load-bearing EXISTS that the optimization deliberately preserves: a partial-coverage distinct reference (with a coverage gap in the target), the same scenario under mixed target/reference encodings to exercise the canonical shift inside the retained EXISTS body, a CTE-shadowed self-reference where target and reference both resolve to the CTE, and a subquery reference whose filter induces partial coverage. Each of these would silently return extra rows if EXISTS were wrongly skipped on a non-self path. --- .../test_disjoin_coordinate_space.py | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) diff --git a/tests/integration/coordinate_space/test_disjoin_coordinate_space.py b/tests/integration/coordinate_space/test_disjoin_coordinate_space.py index e15e58a..668249c 100644 --- a/tests/integration/coordinate_space/test_disjoin_coordinate_space.py +++ b/tests/integration/coordinate_space/test_disjoin_coordinate_space.py @@ -112,3 +112,202 @@ def test_disjoin_should_yield_target_encoded_subintervals_when_reference_differs encode(10, 30, coordinate_system, interval_type), ] assert result == expected + + @pytest.mark.parametrize( + ("coordinate_system", "interval_type"), + CONVENTIONS, + ids=lambda v: str(v), + ) + def test_disjoin_self_mode_and_explicit_self_reference_yield_identical_partitions( + self, giql_query, coordinate_system, interval_type + ): + """Test omitted-reference and explicit-self-reference produce identical rows. + + Given: + Two overlapping intervals (canonical [0, 20) and [10, 30)) + re-encoded under one convention. + When: + ``DISJOIN(features)`` and ``DISJOIN(features, reference := features)`` + are unioned (tagged) in one query. + Then: + Both tag groups should return the same partition + {[0, 10), [10, 20), [20, 30)} re-encoded under the target + convention — proving the two self-reference paths take the same + optimized branch. + """ + # Arrange + s_a, e_a = encode(0, 20, coordinate_system, interval_type) + s_b, e_b = encode(10, 30, coordinate_system, interval_type) + tables = [make_table("features", coordinate_system, interval_type)] + rows = [_row("chr1", s_a, e_a, "a"), _row("chr1", s_b, e_b, "b")] + expected_partition = [ + ("chr1", *encode(0, 10, coordinate_system, interval_type)), + ("chr1", *encode(10, 20, coordinate_system, interval_type)), + ("chr1", *encode(20, 30, coordinate_system, interval_type)), + ] + + # Act + result = giql_query( + "SELECT 'omitted' AS tag, disjoin_chrom, disjoin_start, disjoin_end " + "FROM DISJOIN(features) " + "UNION ALL " + "SELECT 'explicit' AS tag, disjoin_chrom, disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := features) " + "ORDER BY tag, disjoin_start", + tables=tables, + features=rows, + ) + + # Assert + omitted = sorted({row[1:] for row in result if row[0] == "omitted"}) + explicit = sorted({row[1:] for row in result if row[0] == "explicit"}) + assert omitted == expected_partition + assert explicit == expected_partition + + def test_disjoin_drops_segments_outside_partial_reference_coverage(self, giql_query): + """Test DISJOIN drops sub-intervals not covered by the reference set. + + Given: + A target interval [0, 100) and a reference set covering only + [0, 30) and [70, 100) (gap [30, 70)), canonical 0-based half-open. + When: + DISJOIN(features, reference := refs) runs. + Then: + The result should contain exactly {[0, 30), [70, 100)} and the + uncovered [30, 70) segment should be dropped — proving the EXISTS + coverage filter is preserved for distinct references. + """ + # Arrange + tables = [ + make_table("features", "0based", "half_open"), + make_table("refs", "0based", "half_open"), + ] + + # Act + result = giql_query( + "SELECT DISTINCT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", + tables=tables, + features=[_row("chr1", 0, 100, "t")], + refs=[_row("chr1", 0, 30, "r1"), _row("chr1", 70, 100, "r2")], + ) + + # Assert + assert result == [(0, 30), (70, 100)] + + def test_disjoin_drops_segments_outside_partial_reference_coverage_when_target_and_reference_use_different_encodings( + self, giql_query + ): + """Test EXISTS canonicalization across mixed target/reference encodings. + + Given: + A target interval canonical [0, 100) stored 0-based half-open and + a partial-coverage reference {[0, 30), [70, 100)} stored 1-based + closed. + When: + DISJOIN(features, reference := refs) runs. + Then: + The result should contain the same canonical surviving sub-intervals + {[0, 30), [70, 100)} re-encoded under the target convention — the + EXISTS body still canonicalizes the reference endpoints correctly. + """ + # Arrange + target_system, target_type = "0based", "half_open" + ref_system, ref_type = "1based", "closed" + s_t, e_t = encode(0, 100, target_system, target_type) + r1s, r1e = encode(0, 30, ref_system, ref_type) + r2s, r2e = encode(70, 100, ref_system, ref_type) + tables = [ + make_table("features", target_system, target_type), + make_table("refs", ref_system, ref_type), + ] + + # Act + result = giql_query( + "SELECT DISTINCT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := refs) ORDER BY disjoin_start", + tables=tables, + features=[_row("chr1", s_t, e_t, "t")], + refs=[_row("chr1", r1s, r1e, "r1"), _row("chr1", r2s, r2e, "r2")], + ) + + # Assert + expected = [ + encode(0, 30, target_system, target_type), + encode(70, 100, target_system, target_type), + ] + assert result == expected + + def test_disjoin_keeps_coverage_filter_when_cte_shadows_target_name( + self, giql_query + ): + """Test CTE-shadowed self-reference still applies the coverage filter. + + Given: + A registered ``features`` table with two intervals and an outer + WITH clause defining a CTE ``features`` that is a strict subset + (one row). + When: + ``WITH features AS (SELECT * FROM features WHERE name = 'a') + SELECT ... FROM DISJOIN(features, reference := features)`` is + unioned (tagged) with the registered-table self-mode partition. + Then: + The CTE-shadowed result should be just {[0, 50)} (one segment), + strictly smaller than the registered-table partition + {[0, 30), [30, 50), [50, 100)} — the CTE shadows both target and + reference and EXISTS remains active. + """ + # Arrange + tables = [make_table("features", "0based", "half_open")] + rows = [_row("chr1", 0, 50, "a"), _row("chr1", 30, 100, "b")] + + # Act + result = giql_query( + "SELECT 'shadowed' AS tag, disjoin_chrom, disjoin_start, disjoin_end " + "FROM (WITH features AS (SELECT * FROM features WHERE name = 'a') " + "SELECT * FROM DISJOIN(features, reference := features)) AS s " + "UNION ALL " + "SELECT 'unshadowed' AS tag, disjoin_chrom, disjoin_start, disjoin_end " + "FROM DISJOIN(features) " + "ORDER BY tag, disjoin_start", + tables=tables, + features=rows, + ) + + # Assert + shadowed = sorted({row[1:] for row in result if row[0] == "shadowed"}) + unshadowed = sorted({row[1:] for row in result if row[0] == "unshadowed"}) + assert shadowed == [("chr1", 0, 50)] + assert unshadowed == [("chr1", 0, 30), ("chr1", 30, 50), ("chr1", 50, 100)] + + def test_disjoin_keeps_coverage_filter_when_reference_is_partial_coverage_subquery( + self, giql_query + ): + """Test a partial-coverage subquery reference still drops uncovered segments. + + Given: + A target with one wide row [0, 100) and one narrow row [20, 50) + on chr1; a subquery reference selecting only rows with + ``"start" >= 20`` (which yields only the narrow row). + When: + ``DISJOIN(features, reference := (SELECT * FROM features WHERE + "start" >= 20))`` runs. + Then: + Only the [20, 50) sub-interval should survive — proving subquery + references retain the EXISTS coverage filter end-to-end. + """ + # Arrange + tables = [make_table("features", "0based", "half_open")] + + # Act + result = giql_query( + "SELECT DISTINCT disjoin_start, disjoin_end " + "FROM DISJOIN(features, reference := (" + 'SELECT * FROM features WHERE "start" >= 20' + ")) ORDER BY disjoin_start", + tables=tables, + features=[_row("chr1", 0, 100, "wide"), _row("chr1", 20, 50, "partial")], + ) + + # Assert + assert result == [(20, 50)] From 822cdb0836925035dbda9ad0307eba19a63dec60 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 20 May 2026 13:28:10 -0400 Subject: [PATCH 022/142] refactor: Build DISJOIN WHERE clause from a clauses list Replace the inline exists_clause string-with-leading-space ternary with a where_clauses list joined by AND, mirroring the pattern already used in giqlnearest_sql. The previous form was one missing space away from emitting a syntactically broken WHERE; the list form removes that fragility and makes the conditional EXISTS appendage explicit. The emitted SQL is character-identical in both self-reference and distinct-reference modes. Also extend the leading comment to note that the __giql_dj_ref CTE itself stays live in self-reference mode because the breakpoint CTE still draws from it, preventing a future reader from trying to prune the seemingly unused reference CTE, and tighten the _resolve_disjoin_reference docstring to make the False semantics of is_self_reference explicit for subquery and CTE references. --- src/giql/generators/base.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 243e613..9e190e2 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -367,23 +367,22 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: # In self-reference mode the coverage EXISTS is provably always true: # every emitted segment lies inside its parent target row, and that # row is itself a member of the reference set. Skip the clause so the - # planner does not waste work on a no-op semi-join. - exists_clause = ( - "" - if is_self_reference - else ( - " AND EXISTS (SELECT 1 FROM __giql_dj_ref AS r " - f'WHERE r."{ref_chrom}" = s.kc AND {r_start} <= s.seg_start ' - f"AND {r_end} > s.seg_start)" + # planner does not waste work on a no-op semi-join. The __giql_dj_ref + # CTE itself stays live because __giql_dj_bp still draws breakpoints + # from it. + where_clauses = ["s.seg_end IS NOT NULL", "s.seg_end > s.seg_start"] + if not is_self_reference: + where_clauses.append( + f'EXISTS (SELECT 1 FROM __giql_dj_ref AS r WHERE r."{ref_chrom}" = s.kc ' + f"AND {r_start} <= s.seg_start AND {r_end} > s.seg_start)" ) - ) + where_sql = " AND ".join(where_clauses) final_select = ( f"SELECT t.*, s.kc AS disjoin_chrom, {out_start} AS disjoin_start, " f"{out_end} AS disjoin_end FROM __giql_dj_tgt AS t " f'JOIN __giql_dj_segs AS s ON t."{target_chrom}" = s.kc ' f'AND t."{target_start}" = s.ks AND t."{target_end}" = s.ke ' - "WHERE s.seg_end IS NOT NULL AND s.seg_end > s.seg_start" - f"{exists_clause}" + f"WHERE {where_sql}" ) return ( f"(WITH {ref_cte}, {tgt_cte}, {bp_cte}, " @@ -951,7 +950,9 @@ def _resolve_disjoin_reference( reference provably resolves to the same registered relation as the target (omitted reference, or a bare name equal to the target name that is not shadowed by an enclosing CTE and maps to the same - registered table). + registered table). It is ``False`` in all other cases, including + subquery and CTE references that may textually duplicate the + target. :raises ValueError: If the reference is not a table name, a CTE, or a subquery, or if a bare name resolves to neither a registered table nor a CTE. From 4d6a7db903c7ce417b36b1fff07ff5faf8e44d7f Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 20 May 2026 13:28:54 -0400 Subject: [PATCH 023/142] test: Strengthen DISJOIN EXISTS-skip assertions and rename outlier tests The nested-CTE-shadow transpilation test asserted only that EXISTS appeared somewhere in the output. Tighten it to assert that exactly one EXISTS clause appears, matching the assertion style of the sibling mixed-mode test and ruling out a stray second EXISTS leaking in from the outer scope. Rename two integration tests on TestDisjoinCoordinateSpace from should_yield_X to the declarative-present form already used by the other five tests in the class and by every sibling coordinate-space test file. --- .../coordinate_space/test_disjoin_coordinate_space.py | 4 ++-- tests/test_disjoin_transpilation.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/coordinate_space/test_disjoin_coordinate_space.py b/tests/integration/coordinate_space/test_disjoin_coordinate_space.py index 668249c..f1b36d6 100644 --- a/tests/integration/coordinate_space/test_disjoin_coordinate_space.py +++ b/tests/integration/coordinate_space/test_disjoin_coordinate_space.py @@ -30,7 +30,7 @@ class TestDisjoinCoordinateSpace: CONVENTIONS, ids=lambda v: str(v), ) - def test_disjoin_should_yield_the_partition_in_the_target_encoding( + def test_disjoin_yields_the_partition_in_the_target_encoding( self, giql_query, coordinate_system, interval_type ): """Test DISJOIN self-mode yields the partition in the target's encoding. @@ -71,7 +71,7 @@ def test_disjoin_should_yield_the_partition_in_the_target_encoding( CONVENTIONS, ids=lambda v: str(v), ) - def test_disjoin_should_yield_target_encoded_subintervals_when_reference_differs( + def test_disjoin_yields_target_encoded_subintervals_when_reference_differs( self, giql_query, coordinate_system, interval_type ): """Test DISJOIN emits target-encoded sub-intervals across mixed encodings. diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index bfee25a..b9f3e6e 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -531,4 +531,4 @@ def test_giqldisjoin_sql_should_emit_exists_clause_when_enclosing_cte_shadows_ta ) # Assert - assert "EXISTS (" in sql + assert sql.count("EXISTS (") == 1 From 9a89d5d21a04291bbb5522c318dc122a6ca59647 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:00:37 -0400 Subject: [PATCH 024/142] refactor: Move _extract_bool_param into private-methods region of BaseGIQLGenerator The private static helper was placed before __init__, violating the class-member ordering convention where __init__/__new__ precede private methods. Move it next to the other private static helpers (_enclosing_cte_names) where readers expect private utilities to live, and tighten its docstring's exp.Boolean / exp.Literal references with :class: role markup. --- src/giql/generators/base.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 9e190e2..d8e9923 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -35,19 +35,6 @@ class BaseGIQLGenerator(Generator): # SQLite does not support LATERAL, so it overrides this to False SUPPORTS_LATERAL = True - @staticmethod - def _extract_bool_param(param_expr: exp.Expression | None) -> bool: - """Extract boolean value from a parameter expression. - - Handles exp.Boolean, exp.Literal, and string representations. - """ - if not param_expr: - return False - elif isinstance(param_expr, exp.Boolean): - return param_expr.this - else: - return str(param_expr).upper() in ("TRUE", "1", "YES") - def __init__(self, tables: Tables | None = None, **kwargs): super().__init__(**kwargs) self.tables = tables or Tables() @@ -1026,6 +1013,20 @@ def _resolve_disjoin_reference( "the CTE before transpiling." ) + @staticmethod + def _extract_bool_param(param_expr: exp.Expression | None) -> bool: + """Extract boolean value from a parameter expression. + + Handles :class:`exp.Boolean`, :class:`exp.Literal`, and string + representations. + """ + if not param_expr: + return False + elif isinstance(param_expr, exp.Boolean): + return param_expr.this + else: + return str(param_expr).upper() in ("TRUE", "1", "YES") + @staticmethod def _enclosing_cte_names(expression: exp.Expression) -> set[str]: """Collect CTE names visible to an expression from enclosing WITH clauses. From ba5a59a35c56044598f3b3679e8dccb91db8c174 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:01:42 -0400 Subject: [PATCH 025/142] style: Adopt Python Style Guide conventions in IntersectsBinnedJoinTransformer Two small drive-by fixes flagged during the PR #93 style review: - The __init__ docstring referenced DEFAULT_BIN_SIZE as bare text; switch to :data:`~giql.constants.DEFAULT_BIN_SIZE` so the reference resolves to an anchor in generated docs. - Five inline comments inside _build_pairs_replacement_joins ("Determine which INTERSECTS side maps to FROM vs JOIN table", "Ensure key-only bin CTEs exist", "Build and attach the pairs CTE", "join1: [SIDE] JOIN pairs ON from.key = pairs.from_key", and "join2: [SIDE] JOIN join_table ON join.key = pairs.join_key") restated what the next line of code did rather than explaining why. The four follow-on what-comments are dropped; the prefix-assignment block keeps a single why-comment that captures the symmetry invariant the code preserves. No behavior change. --- src/giql/transformer.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/giql/transformer.py b/src/giql/transformer.py index ed0b3e1..619f419 100644 --- a/src/giql/transformer.py +++ b/src/giql/transformer.py @@ -644,7 +644,7 @@ def __init__(self, tables: Tables, bin_size: int | None = None): Table configurations for column mapping :param bin_size: Bin width for the equi-join rewrite. Defaults to - DEFAULT_BIN_SIZE if not specified. + :data:`~giql.constants.DEFAULT_BIN_SIZE` if not specified. """ self.tables = tables resolved = bin_size if bin_size is not None else DEFAULT_BIN_SIZE @@ -842,7 +842,9 @@ def _build_pairs_replacement_joins( from_cols = self._get_columns(from_table_name) join_cols = self._get_columns(join_table_name) - # Determine which INTERSECTS side maps to FROM vs JOIN table + # Prefix assignment encodes which INTERSECTS argument feeds the + # left vs right bin column of the pairs CTE, so the downstream + # equi-join compares matching halves regardless of FROM order. if left_alias == from_alias: l_table_name, r_table_name = from_table_name, join_table_name l_cols, r_cols = from_cols, join_cols @@ -852,11 +854,9 @@ def _build_pairs_replacement_joins( l_cols, r_cols = join_cols, from_cols from_prefix, join_prefix = "__giql_r", "__giql_l" - # Ensure key-only bin CTEs exist l_cte = self._ensure_key_binned(query, l_table_name, key_binned) r_cte = self._ensure_key_binned(query, r_table_name, key_binned) - # Build and attach the pairs CTE pairs_name = f"__giql_pairs_{pairs_idx}" pairs_cte = self._build_pairs_cte(pairs_name, l_cte, r_cte, l_cols, r_cols) existing_with = query.args.get("with_") @@ -868,7 +868,6 @@ def _build_pairs_replacement_joins( side = join.args.get("side") p_alias = f"__giql_p{pairs_idx}" - # join1: [SIDE] JOIN pairs ON from.key = pairs.from_key join1_on = self._build_key_match(from_alias, from_cols, p_alias, from_prefix) join1_kwargs: dict = { "this": exp.Table( @@ -881,7 +880,6 @@ def _build_pairs_replacement_joins( join1_kwargs["side"] = side join1 = exp.Join(**join1_kwargs) - # join2: [SIDE] JOIN join_table ON join.key = pairs.join_key join2_on = self._build_key_match(join_alias, join_cols, p_alias, join_prefix) if extra: join2_on = exp.And(this=join2_on, expression=extra) From e298609ede8771f9a943314806aeab83dd8dc941 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:02:15 -0400 Subject: [PATCH 026/142] refactor: Extract coordinate canonicalization to giql.canonical module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The canonical-start / canonical-end conversions and their inverses lived inside BaseGIQLGenerator as private methods. With the DuckDB IEJoin dialect transformer about to land — which also needs to emit 0-based half-open SQL regardless of how each source table declares its coordinate system — the conversion logic needs to be shared across both surfaces. Lift canonical_start, canonical_end, decanonical_start, and decanonical_end out of BaseGIQLGenerator into a new module-level giql.canonical surface. Each function takes a raw column expression plus a Table (or None) and wraps the expression with whatever offset the table's (coordinate_system, interval_type) pair requires to reach the canonical 0-based half-open form. The decanonical counterparts apply the inverse offset so emitted endpoints land back in the table's own encoding. BaseGIQLGenerator now imports and delegates to the module-level helpers. Behavior is unchanged for every existing call site. --- src/giql/canonical.py | 91 +++++++++++++++++++++ src/giql/generators/base.py | 158 +++++++++++------------------------- 2 files changed, 137 insertions(+), 112 deletions(-) create mode 100644 src/giql/canonical.py diff --git a/src/giql/canonical.py b/src/giql/canonical.py new file mode 100644 index 0000000..f019a42 --- /dev/null +++ b/src/giql/canonical.py @@ -0,0 +1,91 @@ +"""Convert raw column expressions to and from canonical 0-based half-open coordinates. + +Genomic intervals can be expressed in 0-based half-open, 0-based closed, +1-based half-open, or 1-based closed coordinates. The transpiler emits SQL +in 0-based half-open by canonicalizing every column endpoint at the source. +These helpers wrap a raw column reference with the offset needed for that +canonicalization based on the source table's declared coordinate system. +The ``decanonical_*`` helpers apply the inverse offset to emit an endpoint +back in the table's own coordinate system. (Literal ranges go through +:meth:`giql.range_parser.RangeParser.to_zero_based_half_open`.) +""" + +from giql.table import Table + + +def canonical_start(raw_start: str, table: Table | None) -> str: + """Wrap a raw start column expression to canonical 0-based half-open start. + + When ``table`` is ``None``, the raw expression is returned unchanged + (treated as 0-based half-open). + + Conversion by :attr:`Table.coordinate_system`: + + - ``0based``: ``start`` (identity) + - ``1based``: ``start - 1`` + """ + if table is None or table.coordinate_system == "0based": + return raw_start + return f"({raw_start} - 1)" + + +def canonical_end(raw_end: str, table: Table | None) -> str: + """Wrap a raw end column expression to canonical 0-based half-open end. + + When ``table`` is ``None``, the raw expression is returned unchanged + (treated as 0-based half-open). + + Conversion by (:attr:`Table.coordinate_system`, :attr:`Table.interval_type`): + + - ``0based`` / ``half_open``: ``end`` (identity) + - ``0based`` / ``closed``: ``end + 1`` + - ``1based`` / ``half_open``: ``end - 1`` + - ``1based`` / ``closed``: ``end`` (identity) + """ + if table is None: + return raw_end + key = (table.coordinate_system, table.interval_type) + if key == ("0based", "closed"): + return f"({raw_end} + 1)" + if key == ("1based", "half_open"): + return f"({raw_end} - 1)" + return raw_end + + +def decanonical_start(canonical_expr: str, table: Table | None) -> str: + """Convert a canonical 0-based half-open start to the table's encoding. + + Inverse of :func:`canonical_start`. When ``table`` is ``None``, the + expression is returned unchanged (treated as 0-based half-open). + + Conversion by :attr:`Table.coordinate_system`: + + - ``0based``: ``start`` (identity) + - ``1based``: ``start + 1`` + """ + if table is None or table.coordinate_system == "0based": + return canonical_expr + return f"({canonical_expr} + 1)" + + +def decanonical_end(canonical_expr: str, table: Table | None) -> str: + """Convert a canonical 0-based half-open end to the table's encoding. + + Inverse of :func:`canonical_end`. When ``table`` is ``None``, the + expression is returned unchanged (treated as 0-based half-open). + + Conversion by (:attr:`Table.coordinate_system`, :attr:`Table.interval_type`): + + - ``0based`` / ``half_open``: ``end`` (identity) + - ``0based`` / ``closed``: ``end - 1`` + - ``1based`` / ``half_open``: ``end + 1`` + - ``1based`` / ``closed``: ``end`` (identity) + """ + if table is None: + return canonical_expr + key = (table.coordinate_system, table.interval_type) + if key == ("0based", "closed"): + return f"({canonical_expr} - 1)" + if key == ("1based", "half_open"): + return f"({canonical_expr} + 1)" + return canonical_expr diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index d8e9923..ccdcf08 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -1,6 +1,10 @@ from sqlglot import exp from sqlglot.generator import Generator +from giql.canonical import canonical_end +from giql.canonical import canonical_start +from giql.canonical import decanonical_end +from giql.canonical import decanonical_start from giql.constants import DEFAULT_CHROM_COL from giql.constants import DEFAULT_END_COL from giql.constants import DEFAULT_START_COL @@ -187,12 +191,10 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: target_strand = f'{table_name}."{target_table.strand_col}"' # Distance math below assumes 0-based half-open. - target_start_expr = self._canonical_start( + target_start_expr = canonical_start( f'{table_name}."{target_start}"', target_table ) - target_end_expr = self._canonical_end( - f'{table_name}."{target_end}"', target_table - ) + target_end_expr = canonical_end(f'{table_name}."{target_end}"', target_table) # Build distance calculation using CASE expression # For NEAREST: ORDER BY absolute distance, but RETURN signed distance @@ -272,6 +274,10 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: filter drops sub-intervals overlapping no reference interval. When no ``reference`` is given it defaults to the target set. + Coordinate-system round-tripping is handled by + :func:`giql.canonical.decanonical_start` / + :func:`giql.canonical.decanonical_end`. + :param expression: GIQLDisjoin expression node :return: @@ -304,21 +310,21 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: # Canonical target endpoints, qualified by the __giql_dj_tgt alias. t_chrom = f't."{target_chrom}"' - t_start = self._canonical_start(f't."{target_start}"', target_table) - t_end = self._canonical_end(f't."{target_end}"', target_table) + t_start = canonical_start(f't."{target_start}"', target_table) + t_end = canonical_end(f't."{target_end}"', target_table) # Canonical reference endpoints: unqualified for the breakpoint CTE, # qualified by 'r' for the coverage EXISTS filter. - bp_start = self._canonical_start(f'"{ref_start}"', ref_table) - bp_end = self._canonical_end(f'"{ref_end}"', ref_table) - r_start = self._canonical_start(f'r."{ref_start}"', ref_table) - r_end = self._canonical_end(f'r."{ref_end}"', ref_table) + bp_start = canonical_start(f'"{ref_start}"', ref_table) + bp_end = canonical_end(f'"{ref_end}"', ref_table) + r_start = canonical_start(f'r."{ref_start}"', ref_table) + r_end = canonical_end(f'r."{ref_end}"', ref_table) # disjoin_start / disjoin_end are emitted in the target table's # coordinate system so an output row carries one convention; the cut # math above stays canonical internally. - out_start = self._decanonical_start("s.seg_start", target_table) - out_end = self._decanonical_end("s.seg_end", target_table) + out_start = decanonical_start("s.seg_start", target_table) + out_end = decanonical_end("s.seg_end", target_table) # Build the WITH clause one named fragment per __giql_dj_* CTE so each # block reads on its own. The `seg_end > seg_start` guard in the final @@ -425,10 +431,10 @@ def giqldistance_sql(self, expression: GIQLDistance) -> str: # Distance math below assumes 0-based half-open. table_a = self._resolve_table(interval_a_sql) table_b = self._resolve_table(interval_b_sql) - start_a = self._canonical_start(start_a, table_a) - end_a = self._canonical_end(end_a, table_a) - start_b = self._canonical_start(start_b, table_b) - end_b = self._canonical_end(end_b, table_b) + start_a = canonical_start(start_a, table_a) + end_a = canonical_end(end_a, table_a) + start_b = canonical_start(start_b, table_b) + end_b = canonical_end(end_b, table_b) # Generate CASE expression return self._generate_distance_case( @@ -585,8 +591,8 @@ def _generate_range_predicate( column_ref, self._current_table ) table = self._resolve_table(column_ref, self._current_table) - start_col = self._canonical_start(raw_start_col, table) - end_col = self._canonical_end(raw_end_col, table) + start_col = canonical_start(raw_start_col, table) + end_col = canonical_end(raw_end_col, table) chrom = parsed_range.chromosome start = parsed_range.start @@ -644,10 +650,10 @@ def _generate_column_join(self, left_col: str, right_col: str, op_type: str) -> r_chrom, raw_r_start, raw_r_end = self._get_column_refs(right_col, None) l_table = self._resolve_table(left_col) r_table = self._resolve_table(right_col) - l_start = self._canonical_start(raw_l_start, l_table) - l_end = self._canonical_end(raw_l_end, l_table) - r_start = self._canonical_start(raw_r_start, r_table) - r_end = self._canonical_end(raw_r_end, r_table) + l_start = canonical_start(raw_l_start, l_table) + l_end = canonical_end(raw_l_end, l_table) + r_start = canonical_start(raw_r_start, r_table) + r_end = canonical_end(raw_r_end, r_table) if op_type == "intersects": # Ranges overlap if: chrom1 = chrom2 AND start1 < end2 AND end1 > start2 @@ -795,8 +801,9 @@ def _resolve_nearest_reference( Tuple of (chromosome, start, end) or (chromosome, start, end, strand) Returns SQL expressions (column refs for correlated, literals for standalone) Endpoints are canonicalized to 0-based half-open: literal references via - ``RangeParser.to_zero_based_half_open``, column references via - ``_canonical_start`` / ``_canonical_end``. + :meth:`~giql.range_parser.RangeParser.to_zero_based_half_open`, column + references via :func:`giql.canonical.canonical_start` / + :func:`giql.canonical.canonical_end`. :raises ValueError: If reference is missing in standalone mode or invalid format """ @@ -832,7 +839,11 @@ def _resolve_nearest_reference( # Column reference - resolve and canonicalize chrom, start, end = self._get_column_refs(reference_sql, None) ref_table = self._resolve_table(reference_sql) - return self._canonical_endpoints(chrom, start, end, ref_table) + return ( + chrom, + canonical_start(start, ref_table), + canonical_end(end, ref_table), + ) else: # correlated mode if reference: @@ -840,7 +851,11 @@ def _resolve_nearest_reference( reference_sql = self.sql(reference) chrom, start, end = self._get_column_refs(reference_sql, None) ref_table = self._resolve_table(reference_sql) - return self._canonical_endpoints(chrom, start, end, ref_table) + return ( + chrom, + canonical_start(start, ref_table), + canonical_end(end, ref_table), + ) else: # Implicit reference - resolve from outer table in LATERAL join outer_table = self._find_outer_table_in_lateral_join(expression) @@ -864,7 +879,11 @@ def _resolve_nearest_reference( # Build column references using the outer table and genomic column reference_sql = f"{outer_table}.{table.genomic_col}" chrom, start, end = self._get_column_refs(reference_sql, None) - return self._canonical_endpoints(chrom, start, end, table) + return ( + chrom, + canonical_start(start, table), + canonical_end(end, table), + ) def _resolve_target_table( self, expression: GIQLNearest | GIQLDisjoin @@ -1106,91 +1125,6 @@ def _get_column_refs( return base_cols + (f'"{strand_col}"',) return base_cols - @staticmethod - def _canonical_start(raw_start: str, table: Table | None) -> str: - """Wrap a raw start column expression to yield canonical 0-based half-open start. - - Conversion by ``coordinate_system``: - - - 0based: ``start`` (identity) - - 1based: ``start - 1`` - """ - if table is None or table.coordinate_system == "0based": - return raw_start - return f"({raw_start} - 1)" - - @staticmethod - def _canonical_end(raw_end: str, table: Table | None) -> str: - """Wrap a raw end column expression to yield canonical 0-based half-open end. - - Conversion by ``(coordinate_system, interval_type)``: - - - 0based / half_open: ``end`` (identity) - - 0based / closed: ``end + 1`` - - 1based / half_open: ``end - 1`` - - 1based / closed: ``end`` (identity) - """ - if table is None: - return raw_end - key = (table.coordinate_system, table.interval_type) - if key == ("0based", "closed"): - return f"({raw_end} + 1)" - if key == ("1based", "half_open"): - return f"({raw_end} - 1)" - return raw_end # 0based/half_open and 1based/closed: identity - - @staticmethod - def _decanonical_start(canonical_start: str, table: Table | None) -> str: - """Convert a canonical 0-based half-open start to the table's encoding. - - Inverse of ``_canonical_start``. Conversion by ``coordinate_system``: - - - 0based: ``start`` (identity) - - 1based: ``start + 1`` - """ - if table is None or table.coordinate_system == "0based": - return canonical_start - return f"({canonical_start} + 1)" - - @staticmethod - def _decanonical_end(canonical_end: str, table: Table | None) -> str: - """Convert a canonical 0-based half-open end to the table's encoding. - - Inverse of ``_canonical_end``. Conversion by ``(coordinate_system, - interval_type)``: - - - 0based / half_open: ``end`` (identity) - - 0based / closed: ``end - 1`` - - 1based / half_open: ``end + 1`` - - 1based / closed: ``end`` (identity) - """ - if table is None: - return canonical_end - key = (table.coordinate_system, table.interval_type) - if key == ("0based", "closed"): - return f"({canonical_end} - 1)" - if key == ("1based", "half_open"): - return f"({canonical_end} + 1)" - return canonical_end # 0based/half_open and 1based/closed: identity - - @staticmethod - def _canonical_endpoints( - chrom: str, - raw_start: str, - raw_end: str, - table: Table | None, - ) -> tuple[str, str, str]: - """Canonicalize a ``(chrom, start, end)`` triple to 0-based half-open. - - The chromosome expression passes through unchanged; start and end are - wrapped via ``_canonical_start`` / ``_canonical_end``. - """ - return ( - chrom, - BaseGIQLGenerator._canonical_start(raw_start, table), - BaseGIQLGenerator._canonical_end(raw_end, table), - ) - def _resolve_table_name(self, column_ref: str, table_name: str | None) -> str | None: """Resolve the underlying table name for a column reference. From 2d49765e9ad2d300d1760a787ecc6e69092d8fea Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:02:48 -0400 Subject: [PATCH 027/142] feat: Add IntersectsDuckDBIEJoinTransformer for column-to-column INTERSECTS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The binned equi-join plan that IntersectsBinnedJoinTransformer emits generalises across every SQL backend, but on DuckDB it loses to a per-chromosome dynamic-SQL plan that lets DuckDB pick from its range- join family (IE_JOIN / PIECEWISE_MERGE_JOIN). The new transformer emits a SET VARIABLE block whose value is a per-chromosome UNION ALL of pure inequality-predicate subqueries, then selects through query(getvariable(...)). This avoids the cross-chromosome candidate- pair materialization that hurts a global IEJoin and the UNNEST / DISTINCT overhead that hurts the binned plan. The transformer handles INNER, SEMI, and ANTI joins with exactly one column-to-column INTERSECTS predicate plus a broad set of common decorations: ORDER BY / LIMIT / OFFSET / DISTINCT on the outer SELECT, GROUP BY / HAVING with aggregate functions (COUNT, SUM, MIN, MAX, AVG, COUNT(DISTINCT), and any exp.AggFunc subclass), and extra non- INTERSECTS predicates ANDed onto the join ON or WHERE (both single- side and cross-side, including BETWEEN, IN (literal-list), LIKE, IS NULL). Unsupported shapes — outer-join INTERSECTS, self-joins, multi- INTERSECTS queries, three-or-more-table FROM clauses, top-level WITH / CTE clauses, DISTINCT ON, OR/NOT/paren-wrapped extras, unqualified-column extras, extras containing subqueries / aggregates / window functions, and subqueries inside GROUP BY / HAVING / ORDER BY — fall back to the binned plan so callers can opt in unconditionally without hitting silent correctness regressions. Hard-error projection shapes (bare SELECT *, unqualified columns, expression projections, unknown table qualifiers, a.* AS x, window aggregates, FILTER clauses, arithmetic-over-aggregate, scalar subqueries) raise ValueError with a dedicated message naming the offending form. Empty chromosome intersections route through a COALESCE empty-schema fallback so DuckDB receives a typed empty result instead of a runtime NULL crash. Chromosome literals are escaped manually because DuckDB has no quote_literal. Five INTERSECTS-walking helpers (_is_column_intersects, _find_column_intersects_in, _has_outer_join_intersects, _strip_intersects, _count_column_intersects) move out of IntersectsBinnedJoinTransformer to module scope so both transformers reuse one definition. --- src/giql/transformer.py | 1668 ++++++++++++++++++++++++++++----------- 1 file changed, 1225 insertions(+), 443 deletions(-) diff --git a/src/giql/transformer.py b/src/giql/transformer.py index 619f419..b0355c6 100644 --- a/src/giql/transformer.py +++ b/src/giql/transformer.py @@ -5,10 +5,15 @@ SQL with CTEs. """ -import itertools +from dataclasses import dataclass +from typing import ClassVar +from typing import Literal +from uuid import uuid4 from sqlglot import exp +from giql.canonical import canonical_end +from giql.canonical import canonical_start from giql.constants import DEFAULT_BIN_SIZE from giql.constants import DEFAULT_CHROM_COL from giql.constants import DEFAULT_END_COL @@ -17,9 +22,212 @@ from giql.expressions import GIQLCluster from giql.expressions import GIQLMerge from giql.expressions import Intersects +from giql.table import Table from giql.table import Tables +class _UnqualifiedProjectionError(Exception): + """Signal an unrecoverable projection-resolution failure in the DuckDB IEJoin path. + + Raised by :class:`IntersectsDuckDBIEJoinTransformer` when a query engages + the dialect path but contains a SELECT-list shape the dialect cannot + translate (bare ``*``, unqualified column, expression-form projection, + unknown table qualifier, ``a.* AS x``, window aggregate, ``FILTER`` clause, + arithmetic over an aggregate, or a scalar subquery in the projection list), + or an extras predicate that references an unqualified or unknown-aliased + column. Caught and re-raised as :class:`ValueError` at the dialect's public + boundary. + """ + + +# These predicates are shared between :class:`IntersectsBinnedJoinTransformer` +# and :class:`IntersectsDuckDBIEJoinTransformer`; they live at module scope +# rather than on either class so neither owns them. Style §2's strict +# "functions after classes" ordering is relaxed here under the +# topic-interleaved exception, generalized to "shared by multiple consumer +# classes." + + +def _is_column_intersects(node: Intersects) -> bool: + """Return True if *node* is column-to-column :class:`Intersects`. + + Both operands must be table-qualified column references. + """ + return ( + isinstance(node.this, exp.Column) + and bool(node.this.table) + and isinstance(node.expression, exp.Column) + and bool(node.expression.table) + ) + + +def _find_column_intersects_in(expr: exp.Expression) -> Intersects | None: + """Return the first column-to-column :class:`Intersects` in *expr*, or ``None``.""" + return next( + (n for n in expr.find_all(Intersects) if _is_column_intersects(n)), + None, + ) + + +def _normalize_alias(name: str, *, quoted: bool = False) -> str: + """Return *name* case-folded unless it was a quoted identifier. + + DuckDB (and standard SQL) treat unquoted identifiers case-insensitively. + Comparing aliases via raw Python equality rejects valid mixed-case queries + that the binned plan accepts; normalize via :py:meth:`str.casefold` so the + dialect matches DuckDB's identifier semantics. Quoted identifiers stay + case-sensitive per the SQL standard. + """ + return name if quoted else name.casefold() + + +def _has_outer_join_intersects(query: exp.Select) -> bool: + """Return True if any outer JOIN has a column-to-column :class:`Intersects`. + + Handles two surfaces the user can write the INTERSECTS under an outer + join: directly in the join's ``ON`` clause, or in the top-level + ``WHERE`` while the join keeps its ``LEFT``/``RIGHT``/``FULL`` side + modifier (``LEFT JOIN ... ON TRUE WHERE a.interval INTERSECTS + b.interval``). The dialect's inner-join IEJoin rewrite would silently + drop the outer-join's unmatched-row guarantee in the second shape, + so we fall back whenever an outer-join is present and an + INTERSECTS sits anywhere it could influence the join result. + """ + if not isinstance(query, exp.Select): + return False + outer_joins = [ + join for join in query.args.get("joins") or [] if join.args.get("side") + ] + if not outer_joins: + return False + for join in outer_joins: + on = join.args.get("on") + if on is not None and _find_column_intersects_in(on): + return True + where = query.args.get("where") + if where and _find_column_intersects_in(where.this): + return True + return False + + +def _strip_intersects( + expr: exp.Expression | None, intersects: Intersects +) -> exp.Expression | None: + """Return the parts of an AND tree that aren't the given :class:`Intersects`.""" + if expr is None or expr is intersects: + return None + if isinstance(expr, exp.And): + if expr.this is intersects: + return expr.expression + if expr.expression is intersects: + return expr.this + left = _strip_intersects(expr.this, intersects) + right = _strip_intersects(expr.expression, intersects) + if left is None: + return right + if right is None: + return left + return exp.And(this=left, expression=right) + return expr + + +def _count_column_intersects(query: exp.Select) -> int: + """Count column-to-column :class:`Intersects` predicates anywhere in *query*. + + Walks the full AST so the dialect's "exactly one INTERSECTS" gate cannot + be circumvented by an INTERSECTS hidden inside a subquery (e.g. inside + a SELECT-list scalar subquery or inside ``EXISTS (SELECT ...)``). The + SELECT-list / modifier-clause subquery gates in ``transform_to_sql`` + already reject those shapes, but this widened count is defense-in-depth + against future gate relaxations. + """ + if not isinstance(query, exp.Select): + return 0 + return sum(1 for n in query.find_all(Intersects) if _is_column_intersects(n)) + + +@dataclass(frozen=True) +class _IEJoinSides: + """Resolved two-sided join shape for the DuckDB IEJoin dialect path. + + Bundles the left/right user aliases (normalized via :func:`_normalize_alias`) + with the un-aliased :class:`~sqlglot.expressions.Table` AST nodes and a + ``kind`` discriminator identifying the join variant. The hardcoded inner + aliases ``a`` and ``b`` used inside each per-chromosome subquery are + exposed as :data:`left_inner_alias` and :data:`right_inner_alias`. + + The :attr:`is_left_only_join` property is True for SEMI and ANTI shapes, + which restrict the outer SELECT to left-side projections only. + """ + + left_user_alias: str + right_user_alias: str + left_table: exp.Table + right_table: exp.Table + kind: Literal["INNER", "SEMI", "ANTI"] + + left_inner_alias: ClassVar[str] = "a" + right_inner_alias: ClassVar[str] = "b" + + @property + def is_left_only_join(self) -> bool: + """True when the join's output only references the left side (SEMI / ANTI).""" + return self.kind in ("SEMI", "ANTI") + + @classmethod + def from_intersects( + cls, + from_table: exp.Table, + join: exp.Join, + intersects: Intersects, + ) -> "_IEJoinSides | None": + """Resolve sides, run the FROM-side orientation swap, normalize aliases. + + Returns ``None`` when the join's table operand isn't a base + :class:`~sqlglot.expressions.Table`, when neither side of the + :class:`Intersects` predicate references the FROM table's alias, or + when the join's ``kind`` is outside the supported set + (``INNER`` / ``CROSS`` / ``SEMI`` / ``ANTI``; ``CROSS`` folds to + ``INNER`` because per-chromosome partitioning makes them equivalent + inside the inner subquery). + """ + join_table = join.this + if not isinstance(join_table, exp.Table): + return None + from_alias = _normalize_alias(from_table.alias_or_name) + left_alias = _normalize_alias(intersects.this.table) + right_alias = _normalize_alias(intersects.expression.table) + if left_alias == from_alias: + l_user_alias = from_alias + l_table = from_table + r_user_alias = right_alias + r_table = join_table + elif right_alias == from_alias: + # Inner subquery template hardcodes "a" to the FROM side; swap so + # that contract holds regardless of which operand the user put + # first in INTERSECTS. + l_user_alias = from_alias + l_table = from_table + r_user_alias = left_alias + r_table = join_table + else: + return None + raw_kind = (join.args.get("kind") or "").upper() + if raw_kind in ("", "CROSS"): + kind: Literal["INNER", "SEMI", "ANTI"] = "INNER" + elif raw_kind in ("SEMI", "ANTI"): + kind = raw_kind # type: ignore[assignment] + else: + return None + return cls( + left_user_alias=l_user_alias, + right_user_alias=r_user_alias, + left_table=l_table, + right_table=r_table, + kind=kind, + ) + + class ClusterTransformer: """Transforms queries containing CLUSTER into CTE-based queries. @@ -581,60 +789,49 @@ def _transform_for_merge( class IntersectsBinnedJoinTransformer: - """Transforms column-to-column INTERSECTS into binned equi-joins. - - Handles both explicit JOIN ON and implicit cross-join (WHERE) patterns. - Two rewrite strategies are selected based on the SELECT list: - - **No wildcards** (``SELECT a.chrom, b.start, ...``) — ``__giql_bin`` - cannot appear in the output regardless of CTE content, so the simpler - 1-join full-CTE approach is used: - - WITH __giql_a_binned AS ( - SELECT *, UNNEST(range( - CAST("start" / B AS BIGINT), - CAST(("end" - 1) / B + 1 AS BIGINT) - )) AS __giql_bin FROM peaks - ), - __giql_b_binned AS (...) - SELECT DISTINCT a.chrom, b.start, ... - FROM __giql_a_binned AS a - JOIN __giql_b_binned AS b - ON a."chrom" = b."chrom" AND a.__giql_bin = b.__giql_bin - AND a."start" < b."end" AND a."end" > b."start" + """Transform column-to-column INTERSECTS into binned equi-joins. - **Wildcards present** (``SELECT a.*, b.*``) — ``__giql_bin`` would leak - into ``a.*`` expansion if ``a`` aliases a full-select CTE. A key-only - bridge CTE pattern is used instead, keeping original table references: + Handles both explicit ``JOIN ... ON`` and implicit cross-join (WHERE) + INTERSECTS patterns. Each rewrite emits a pairs CTE that holds the + matching ``(left_key, right_key)`` tuples computed by an INNER binned + join with ``SELECT DISTINCT`` on the join keys, then bridges the + original tables through that pairs CTE:: WITH __giql_peaks_bins AS ( SELECT "chrom", "start", "end", UNNEST(range(...)) AS __giql_bin FROM peaks ), - __giql_genes_bins AS (...) - SELECT DISTINCT a.*, b.* + __giql_genes_bins AS (...), + __giql_pairs_0 AS ( + SELECT DISTINCT + __giql_l. AS __giql_l_chrom, ..., + __giql_r. AS __giql_r_chrom, ... + FROM __giql_peaks_bins AS __giql_l + JOIN __giql_genes_bins AS __giql_r + ON __giql_l. = __giql_r. + AND __giql_l.__giql_bin = __giql_r.__giql_bin + AND __giql_l. < __giql_r. + AND __giql_l. > __giql_r. + ) + SELECT a.*, b.* FROM peaks a - JOIN __giql_peaks_bins __giql_c0 - ON a."chrom" = __giql_c0."chrom" AND a."start" = __giql_c0."start" - AND a."end" = __giql_c0."end" - JOIN __giql_genes_bins __giql_c1 - ON __giql_c0."chrom" = __giql_c1."chrom" - AND __giql_c0.__giql_bin = __giql_c1.__giql_bin + JOIN __giql_pairs_0 AS __giql_p0 + ON a. = __giql_p0.__giql_l_chrom + AND a. = __giql_p0.__giql_l_start + AND a. = __giql_p0.__giql_l_end JOIN genes b - ON b."chrom" = __giql_c1."chrom" AND b."start" = __giql_c1."start" - AND b."end" = __giql_c1."end" - AND a."start" < b."end" AND a."end" > b."start" + ON b. = __giql_p0.__giql_r_chrom + AND b. = __giql_p0.__giql_r_start + AND b. = __giql_p0.__giql_r_end + + Deduplication happens inside the pairs CTE on the join keys, so the + output query does NOT carry a ``SELECT DISTINCT`` — source rows that + differ only in unselected columns are preserved. Outer joins ride + safely on top of this because the bridge-join structure prevents bin + fan-out from creating spurious NULL rows. Literal-range INTERSECTS (e.g., ``WHERE interval INTERSECTS 'chr1:...'``) are left untouched. - - SELECT DISTINCT is added to deduplicate rows produced by multi-bin - matches. This means rows that are identical across every selected - column will be collapsed — include a distinguishing column (e.g., an - id or score) to preserve duplicates that differ only in unselected - columns. The bridge path's key-match joins on ``(chrom, start, - end)`` and may fan out if multiple source rows share those values; - DISTINCT corrects for this. """ def __init__(self, tables: Tables, bin_size: int | None = None): @@ -653,36 +850,35 @@ def __init__(self, tables: Tables, bin_size: int | None = None): self.bin_size = resolved def transform(self, query: exp.Expression) -> exp.Expression: + """Rewrite column-to-column INTERSECTS joins as a pairs-CTE binned plan. + + Walks ``query.args["joins"]`` and the top-level ``WHERE`` for + column-to-column :class:`Intersects` predicates and replaces each one + with a bridge through a freshly-allocated pairs CTE + (``__giql_pairs_``) plus the per-table bin CTEs that feed it. + Non-INTERSECTS predicates are preserved verbatim; literal-range + ``INTERSECTS`` (``WHERE interval INTERSECTS 'chr1:100-200'``) is + left untouched and emitted by the default generator. + + :param query: + Parsed query AST. + :return: + The transformed AST (mutated in place when an INTERSECTS join + is rewritten; returned unchanged for non-``Select`` inputs). + """ if not isinstance(query, exp.Select): return query # The pairs-CTE approach computes matching (left_key, right_key) # pairs via an INNER binned join with DISTINCT on key columns, - # then joins the original tables through the pairs CTE. This + # then joins the original tables through the pairs CTE. This # avoids adding SELECT DISTINCT to the output query, which would # collapse legitimately different source rows that happen to - # have identical selected columns. It also handles outer joins + # have identical selected columns. It also handles outer joins # correctly by preventing bin fan-out from creating spurious # NULL rows. return self._transform_with_pairs(query) - def _select_has_wildcards(self, query: exp.Select) -> bool: - """Return True if any SELECT item is a wildcard (* or table.*).""" - for expr in query.expressions: - if isinstance(expr, exp.Star): - return True - if isinstance(expr, exp.Column) and isinstance(expr.this, exp.Star): - return True - return False - - def _has_outer_join_intersects(self, query: exp.Select) -> bool: - """Return True if any outer JOIN has an INTERSECTS predicate.""" - for join in query.args.get("joins") or []: - if join.args.get("side") and join.args.get("on"): - if self._find_column_intersects_in(join.args["on"]): - return True - return False - def _transform_with_pairs(self, query: exp.Select) -> exp.Select: """Transform INTERSECTS joins using the pairs-CTE approach. @@ -702,9 +898,9 @@ def _transform_with_pairs(self, query: exp.Select) -> exp.Select: for join in joins: on = join.args.get("on") if on: - intersects = self._find_column_intersects_in(on) + intersects = _find_column_intersects_in(on) if intersects: - extra = self._extract_non_intersects(on, intersects) + extra = _strip_intersects(on, intersects) replacement = self._build_pairs_replacement_joins( query, join, intersects, extra, key_binned, pairs_idx ) @@ -716,7 +912,7 @@ def _transform_with_pairs(self, query: exp.Select) -> exp.Select: where = query.args.get("where") if where: - intersects = self._find_column_intersects_in(where.this) + intersects = _find_column_intersects_in(where.this) if intersects: cross_join = self._find_cross_join_for_intersects( query, intersects, new_joins @@ -921,158 +1117,6 @@ def _build_key_match( ), ) - def _transform_full_cte(self, query: exp.Select) -> exp.Select: - joins = query.args.get("joins") or [] - binned: dict[str, tuple[str, str, str]] = {} - rewrote_any = False - - for join in joins: - on = join.args.get("on") - if on: - intersects = self._find_column_intersects_in(on) - if intersects: - self._rewrite_join_on_full_cte(query, join, intersects, binned) - rewrote_any = True - - # Implicit cross-join: FROM a, b WHERE a.interval INTERSECTS b.interval - where = query.args.get("where") - if where: - intersects = self._find_column_intersects_in(where.this) - if intersects: - cross_join = self._find_cross_join_for_intersects( - query, intersects, joins - ) - if cross_join is not None: - self._rewrite_cross_join_full_cte( - query, cross_join, intersects, binned - ) - rewrote_any = True - - if rewrote_any: - query.set("distinct", exp.Distinct()) - - return query - - def _rewrite_join_on_full_cte( - self, - query: exp.Select, - join: exp.Join, - intersects: Intersects, - binned: dict[str, tuple[str, str, str]], - ) -> None: - from_table = query.args["from_"].this - join_table = join.this - if not isinstance(from_table, exp.Table) or not isinstance( - join_table, exp.Table - ): - return - - from_alias, from_cols = self._ensure_table_binned_full( - query, from_table, query.args["from_"], binned - ) - join_alias, join_cols = self._ensure_table_binned_full( - query, join_table, join, binned - ) - - extra = self._extract_non_intersects(join.args.get("on"), intersects) - - equi_join = exp.And( - this=exp.EQ( - this=exp.column(from_cols[0], table=from_alias, quoted=True), - expression=exp.column(join_cols[0], table=join_alias, quoted=True), - ), - expression=exp.EQ( - this=exp.column("__giql_bin", table=from_alias), - expression=exp.column("__giql_bin", table=join_alias), - ), - ) - # Place both equi-join and overlap in ON so LEFT/RIGHT/FULL semantics hold. - new_on = exp.And( - this=equi_join, - expression=self._build_overlap(from_alias, join_alias, from_cols, join_cols), - ) - if extra: - new_on = exp.And(this=new_on, expression=extra) - join.set("on", new_on) - - def _rewrite_cross_join_full_cte( - self, - query: exp.Select, - cross_join: exp.Join, - intersects: Intersects, - binned: dict[str, tuple[str, str, str]], - ) -> None: - from_table = query.args["from_"].this - join_table = cross_join.this - if not isinstance(from_table, exp.Table) or not isinstance( - join_table, exp.Table - ): - return - - from_alias, from_cols = self._ensure_table_binned_full( - query, from_table, query.args["from_"], binned - ) - join_alias, join_cols = self._ensure_table_binned_full( - query, join_table, cross_join, binned - ) - - equi_join = exp.And( - this=exp.EQ( - this=exp.column(from_cols[0], table=from_alias, quoted=True), - expression=exp.column(join_cols[0], table=join_alias, quoted=True), - ), - expression=exp.EQ( - this=exp.column("__giql_bin", table=from_alias), - expression=exp.column("__giql_bin", table=join_alias), - ), - ) - cross_join.set( - "on", - exp.And( - this=equi_join, - expression=self._build_overlap( - from_alias, join_alias, from_cols, join_cols - ), - ), - ) - self._remove_intersects_from_where(query, intersects) - - def _ensure_table_binned_full( - self, - query: exp.Select, - table: exp.Table, - parent: exp.Expression, - binned: dict[str, tuple[str, str, str]], - ) -> tuple[str, tuple[str, str, str]]: - """Create a full SELECT * CTE for *table* if needed; replace ref in *parent*.""" - alias = table.alias or table.name - if alias in binned: - return alias, binned[alias] - - table_name = table.name - cols = self._get_columns(table_name) - cte_name = f"__giql_{alias}_binned" - - cte = exp.CTE( - this=self._build_full_binned_select(table_name, cols), - alias=exp.TableAlias(this=exp.Identifier(this=cte_name)), - ) - existing_with = query.args.get("with_") - if existing_with: - existing_with.append("expressions", cte) - else: - query.set("with_", exp.With(expressions=[cte])) - - parent.set( - "this", - exp.Table( - this=exp.Identifier(this=cte_name), - alias=exp.TableAlias(this=exp.Identifier(this=alias)), - ), - ) - binned[alias] = cols - return alias, cols - def _build_bin_range( self, start: str, end: str ) -> tuple[exp.Expression, exp.Expression]: @@ -1102,95 +1146,6 @@ def _build_bin_range( ) return low, high - def _build_full_binned_select( - self, table_name: str, cols: tuple[str, str, str] - ) -> exp.Select: - """Build ``SELECT *, UNNEST(range(...)) AS __giql_bin FROM ``.""" - _chrom, start, end = cols - low, high = self._build_bin_range(start, end) - - range_fn = exp.Anonymous(this="range", expressions=[low, high]) - unnest_fn = exp.Anonymous(this="UNNEST", expressions=[range_fn]) - bin_alias = exp.Alias( - this=unnest_fn, - alias=exp.Identifier(this="__giql_bin"), - ) - - select = exp.Select() - select.select(exp.Star(), copy=False) - select.select(bin_alias, append=True, copy=False) - select.from_(exp.Table(this=exp.Identifier(this=table_name)), copy=False) - return select - - def _transform_bridge(self, query: exp.Select) -> exp.Select: - joins = query.args.get("joins") or [] - key_binned: dict[str, str] = {} - connector_counter = itertools.count() - new_joins: list[exp.Join] = [] - rewrote_any = False - - for join in joins: - on = join.args.get("on") - if on: - intersects = self._find_column_intersects_in(on) - if intersects: - extra = self._build_join_back_joins( - query, - join, - intersects, - key_binned, - connector_counter, - preserve_kind=True, - ) - new_joins.extend(extra) - rewrote_any = True - continue - new_joins.append(join) - - where = query.args.get("where") - if where: - intersects = self._find_column_intersects_in(where.this) - if intersects: - cross_join = self._find_cross_join_for_intersects( - query, intersects, new_joins - ) - if cross_join is not None: - new_joins.remove(cross_join) - extra = self._build_join_back_joins( - query, - cross_join, - intersects, - key_binned, - connector_counter, - preserve_kind=False, - ) - new_joins.extend(extra) - self._remove_intersects_from_where(query, intersects) - rewrote_any = True - - if rewrote_any: - query.set("joins", new_joins) - query.set("distinct", exp.Distinct()) - - return query - - def _find_column_intersects_in(self, expr: exp.Expression) -> Intersects | None: - """Return the first column-to-column Intersects node in *expr*, or None. - - Only the first match is returned. A single JOIN with multiple - INTERSECTS conditions in its ON clause is not supported; only the - first will be rewritten. - """ - for node in expr.find_all(Intersects): - if ( - isinstance(node.this, exp.Column) - and node.this.table - and isinstance(node.expression, exp.Column) - and node.expression.table - ): - return node - return None - def _find_cross_join_for_intersects( self, query: exp.Select, @@ -1226,32 +1181,12 @@ def _remove_intersects_from_where( where = query.args.get("where") if not where: return - remainder = self._extract_non_intersects(where.this, intersects) + remainder = _strip_intersects(where.this, intersects) if remainder is None: query.set("where", None) else: query.set("where", exp.Where(this=remainder)) - def _extract_non_intersects( - self, expr: exp.Expression | None, intersects: Intersects - ) -> exp.Expression | None: - """Return the parts of an AND tree that are not the INTERSECTS node.""" - if expr is None or expr is intersects: - return None - if isinstance(expr, exp.And): - if expr.this is intersects: - return expr.expression - if expr.expression is intersects: - return expr.this - left = self._extract_non_intersects(expr.this, intersects) - right = self._extract_non_intersects(expr.expression, intersects) - if left is None: - return right - if right is None: - return left - return exp.And(this=left, expression=right) - return expr - def _get_columns(self, table_name: str) -> tuple[str, str, str]: """Return (chrom, start, end) column names for a table.""" table = self.tables.get(table_name) @@ -1294,7 +1229,11 @@ def _find_table_name_for_alias(self, query: exp.Select, alias: str) -> str: def _build_key_only_bins_select( self, table_name: str, cols: tuple[str, str, str] ) -> exp.Select: - """Build ``SELECT chrom, start, end, UNNEST(range(...)) AS __giql_bin FROM table``.""" + """Build the binned-key SELECT for *table_name*. + + Emits ``SELECT chrom, start, end, UNNEST(range(...)) AS __giql_bin + FROM ``. + """ chrom, start, end = cols low, high = self._build_bin_range(start, end) @@ -1339,134 +1278,977 @@ def _ensure_key_binned( key_binned[table_name] = cte_name return cte_name - def _build_join_back_joins( - self, - query: exp.Select, - join: exp.Join, - intersects: Intersects, - key_binned: dict[str, str], - connector_counter: itertools.count, - *, - preserve_kind: bool, - ) -> list[exp.Join]: - """Build three replacement JOINs for one INTERSECTS using the join-back pattern. - join1 is always INNER because it key-matches a table against its - own bin CTE — every row has a corresponding bin entry by - construction, so the join side has no effect. +class IntersectsDuckDBIEJoinTransformer: + """Transform column-to-column INTERSECTS joins into a DuckDB IEJoin pattern. + + Emits a two-step dynamic SQL output: + + 1. ``SET VARIABLE __giql_iejoin_ = COALESCE(, );`` + where ```` is a ``string_agg`` over the chromosome + partition that emits one ``SELECT ... FROM (... WHERE chrom = '') + a JOIN (... WHERE chrom = '') b ON `` per + chromosome, joined by ``UNION ALL``. + 2. ``SELECT FROM query(getvariable('__giql_iejoin_'))`` + + Each per-chromosome subquery uses pure inequality predicates which DuckDB + plans through its range-join family (``IE_JOIN`` or ``PIECEWISE_MERGE_JOIN``). + The dialect supports INNER, SEMI, and ANTI variants — the per-chromosome + join keyword switches accordingly, and SEMI / ANTI restrict the outer + SELECT to left-side projections only. + + See :doc:`/transpilation/performance` for the full join-shape support + matrix, fallback enumeration, and hard-error projection shapes. + + Notes: + + - ``a.*`` / ``b.*`` star projections expand to the genomic columns + declared by the corresponding :class:`~giql.table.Table` config + (chrom / start / end, plus strand when set). This narrows the result + schema relative to the binned plan, which delegates star expansion + to DuckDB and projects every base-table column. Users with additional + non-genomic columns should list them explicitly. + - ``SEMI JOIN`` and ``ANTI JOIN`` outputs are left-side only. Any + ``b.col`` / ``b.*`` reference in the outer SELECT, aggregate + arguments, GROUP BY, HAVING, or ORDER BY raises + :class:`_UnqualifiedProjectionError` (wrapped to :class:`ValueError` + at the public boundary). + - ``ANTI JOIN`` partitions on the left table's distinct chromosomes + only (not the chromosome INTERSECT used by INNER / SEMI) so that + left rows on chromosomes absent from the right table are preserved. + """ + + def __init__(self, tables: Tables): + """Initialize transformer. - join2 and join3 inherit the original join's side (LEFT, RIGHT) - when *preserve_kind* is True. + :param tables: + Table configurations for column mapping. """ - join_table = join.this - if not isinstance(join_table, exp.Table): - return [join] + self.tables = tables - join_alias = join_table.alias or join_table.name - join_table_name = join_table.name + def transform_to_sql(self, query: exp.Expression) -> str | None: + """Return DuckDB IEJoin SQL for *query* if applicable, else None. + + The dialect rewrites the *whole* query as + ``SET VARIABLE __giql_iejoin_ = ...; SELECT ... FROM + query(getvariable(...))``, so any top-level clause :meth:`_build_sql` + does not read would be silently dropped. The check below is a + whitelist: engage the dialect path only for a query that is + exactly ``SELECT FROM [JOIN ] {ON|WHERE} `` with no other top-level clause. + Everything else returns ``None`` so the binned plan handles it. + """ + if not isinstance(query, exp.Select): + return None - left_alias = intersects.this.table - right_alias = intersects.expression.table - other_alias = left_alias if right_alias == join_alias else right_alias - if other_alias == join_alias: - return [join] # can't determine structure + # Top-level WITH (CTEs) still falls back to the binned plan. + # ORDER BY / LIMIT / OFFSET / DISTINCT / GROUP BY / HAVING ride + # on the outer SELECT wrapper. ``DISTINCT ON (...)`` is the one + # DISTINCT variant we don't try to translate — it would interact + # with the outer alias rewrite in ways the binned plan handles + # for free. + if query.args.get("with_"): + return None + distinct_node = query.args.get("distinct") + if distinct_node is not None and distinct_node.args.get("on"): + return None - extra = self._extract_non_intersects(join.args.get("on"), intersects) + if _has_outer_join_intersects(query): + return None - other_table_name = self._find_table_name_for_alias(query, other_alias) - other_cols = self._get_columns(other_table_name) - join_cols = self._get_columns(join_table_name) + if _count_column_intersects(query) != 1: + return None - other_cte = self._ensure_key_binned(query, other_table_name, key_binned) - join_cte = self._ensure_key_binned(query, join_table_name, key_binned) + # The wrapper-relation rewriter is not scope-aware: any subquery in + # GROUP BY / HAVING / ORDER BY (or inside a SELECT-list aggregate + # argument) would have its inner column refs substituted with + # wrapper-relation aliases that don't exist in the subquery's own + # scope. Reject those shapes here. ``EXISTS (SELECT ...)`` parses + # as ``Exists(Select(...))`` without an ``exp.Subquery`` wrapper, + # so we also check ``exp.Select`` directly. Bare scalar subqueries + # in the SELECT list (``SELECT (SELECT SUM(x) FROM ...)``) get a + # targeted error from :meth:`_resolve_projections` rather than a + # silent fallback — only subqueries hidden inside aggregate args + # fall back here. + for clause_key in ("group", "having", "order"): + modifier = query.args.get(clause_key) + if modifier is None: + continue + if ( + modifier.find(exp.Subquery) is not None + or modifier.find(exp.Select) is not None + ): + return None + for sel in query.expressions: + target = sel.this if isinstance(sel, exp.Alias) else sel + while isinstance(target, exp.Paren): + target = target.this + if isinstance(target, exp.AggFunc) and ( + target.find(exp.Subquery) is not None + or target.find(exp.Select) is not None + ): + return None - c0 = f"__giql_c{next(connector_counter)}" - c1 = f"__giql_c{next(connector_counter)}" + # The supported shape connects exactly two tables via one INTERSECTS, + # so the top level has exactly one join (either an explicit JOIN ... + # ON or an implicit comma-join in WHERE). A third comma/CROSS-joined + # table would otherwise be silently dropped by _build_sql. + joins = query.args.get("joins") or [] + if len(joins) != 1: + return None + the_join = joins[0] - join_side = None - if preserve_kind: - join_side = join.args.get("side") + # NATURAL needs schema introspection we don't have at transpile time + # (Table configs only expose chrom/start/end/strand; user columns + # like name/score are invisible). The binned plan defers to DuckDB, + # which does see the schema, so falling back IS the optimal plan. + if the_join.args.get("method"): + return None - # join1: key-match from other_alias to its bin CTE - join1 = exp.Join( - this=exp.Table( - this=exp.Identifier(this=other_cte), - alias=exp.TableAlias(this=exp.Identifier(this=c0)), - ), - on=exp.And( - this=exp.And( - this=exp.EQ( - this=exp.column(other_cols[0], table=other_alias, quoted=True), - expression=exp.column(other_cols[0], table=c0, quoted=True), - ), - expression=exp.EQ( - this=exp.column(other_cols[1], table=other_alias, quoted=True), - expression=exp.column(other_cols[1], table=c0, quoted=True), - ), - ), - expression=exp.EQ( - this=exp.column(other_cols[2], table=other_alias, quoted=True), - expression=exp.column(other_cols[2], table=c0, quoted=True), - ), - ), - ) + # USING admits only the single-column form whose column matches both + # tables' ``chrom_col`` (the per-chromosome partition is exactly the + # equi-join that ``USING(chrom)`` requests). The chrom-equivalence + # check itself happens in ``_build_sql`` after ``l_table`` / + # ``r_table`` are resolved. Multi-column USING and USING(non-chrom) + # fall back; inline support for those is a documented follow-up. + using_cols = the_join.args.get("using") or [] + if using_cols and len(using_cols) != 1: + return None - # join2: bin equi-join (chrom + __giql_bin match) - join2_kwargs: dict = { - "this": exp.Table( - this=exp.Identifier(this=join_cte), - alias=exp.TableAlias(this=exp.Identifier(this=c1)), - ), - "on": exp.And( - this=exp.EQ( - this=exp.column(other_cols[0], table=c0, quoted=True), - expression=exp.column(join_cols[0], table=c1, quoted=True), - ), - expression=exp.EQ( - this=exp.column("__giql_bin", table=c0), - expression=exp.column("__giql_bin", table=c1), - ), - ), - } - if join_side: - join2_kwargs["side"] = join_side - join2 = exp.Join(**join2_kwargs) + # INNER (the default), CROSS (functionally equivalent to INNER once + # the chromosome partition is in place), SEMI, and ANTI all engage. + # Everything else falls back so the binned plan can preserve its + # semantics (e.g. RIGHT / FULL outer joins). + kind_str = the_join.args.get("kind") + if kind_str and kind_str.upper() not in ("INNER", "CROSS", "SEMI", "ANTI"): + return None - # join3: key-match from join CTE back to actual join table + overlap - key_match = exp.And( - this=exp.And( - this=exp.EQ( - this=exp.column(join_cols[0], table=join_alias, quoted=True), - expression=exp.column(join_cols[0], table=c1, quoted=True), - ), - expression=exp.EQ( - this=exp.column(join_cols[1], table=join_alias, quoted=True), - expression=exp.column(join_cols[1], table=c1, quoted=True), - ), - ), - expression=exp.EQ( - this=exp.column(join_cols[2], table=join_alias, quoted=True), - expression=exp.column(join_cols[2], table=c1, quoted=True), - ), + intersects, the_join = self._find_target_join(query) + if intersects is None or the_join is None: + return None + + from_table = query.args["from_"].this + if not isinstance(from_table, exp.Table): + return None + + # Reject non-base-table operands. A GIQL table-function operand such + # as ``DISJOIN(genes)`` parses as an ``exp.Table`` with an empty + # ``name`` and would be interpolated as an empty identifier into + # broken SQL. (A CTE-named operand is already handled by the + # ``with_`` guard above; an unregistered base table is fine — the + # dialect uses default columns just like the binned path does.) + if not from_table.name: + return None + join_table = the_join.this + if not isinstance(join_table, exp.Table) or not join_table.name: + return None + + sides = _IEJoinSides.from_intersects(from_table, the_join, intersects) + if sides is None: + return None + + if sides.left_user_alias == sides.right_user_alias: + # Same alias on both sides of the INTERSECTS — the inner subquery + # would alias the same underlying relation as both "a" and "b", + # which the binned plan handles correctly via its bridge CTE. + return None + + # Compare fully-qualified identifiers (catalog/db/name) so two + # distinct same-named tables in different schemas are not treated + # as a self-join. Same qualified identifier still falls back so the + # binned plan handles the legitimate self-join shape. + if self._qualified_table_ident(sides.left_table) == self._qualified_table_ident( + sides.right_table + ): + return None + + # Extra predicates are inlined into each per-chromosome subquery's + # join ON (see :meth:`_build_sql`). Soft-fallback residuals (subquery, + # aggregate, window, or an INTERSECTS still wrapped in OR/NOT/Paren) + # cause ``_build_sql`` to return None. User-mistake residuals + # (unqualified or unknown-aliased columns, or columns with a + # catalog/schema qualifier) raise :class:`_UnqualifiedProjectionError`, + # which we wrap to :class:`ValueError` here so the public surface is + # uniform. + try: + return self._build_sql(query, intersects, sides, the_join) + except _UnqualifiedProjectionError as exc: + raise ValueError(str(exc)) from exc + + @staticmethod + def _sql_escape(s: str) -> str: + """Return *s* with single quotes doubled for safe SQL string literal use.""" + return s.replace("'", "''") + + @staticmethod + def _sql_quote_ident(name: str) -> str: + """Render *name* as a DuckDB-quoted identifier. + + Delegates to sqlglot's identifier emitter so embedded double quotes + are doubled correctly and DuckDB-specific quoting rules (case, + reserved words) are honored. Used at every identifier-interpolation + site in the dialect's dynamic SQL. + """ + return exp.to_identifier(name, quoted=True).sql(dialect="duckdb") + + @staticmethod + def _qualified_table_ident(table: exp.Table) -> str: + """Render *table*'s catalog/db/name as a qualified DuckDB identifier. + + Drops the user-supplied alias (``peaks AS a``) so the rendered string + is suitable both as a bare ``FROM`` operand (where the caller adds + its own alias) and as a subquery source. Catalog and schema qualifiers + survive — sqlglot's :meth:`~sqlglot.expressions.Expression.sql` emitter + handles identifier quoting per DuckDB's rules (reserved words and + special characters get quoted; safe unquoted identifiers stay + unquoted). + """ + no_alias = table.copy() + no_alias.set("alias", None) + return no_alias.sql(dialect="duckdb") + + @staticmethod + def _classify_extras( + extras: list[exp.Expression], + ) -> Literal["inline", "fallback"]: + """Return ``"fallback"`` if any extra has a shape the binned plan must handle. + + Soft-fallback shapes: an INTERSECTS still embedded in the residual + (because :func:`_strip_intersects` couldn't peel an OR/NOT/Paren + wrapper), a subquery, an aggregate function, or a window function. + Returns ``"inline"`` when every extra is safe to inline into the + per-chromosome subquery's join ON clause. + """ + for predicate in extras: + if predicate.find(Intersects) is not None: + return "fallback" # INTERSECTS wrapped in OR/NOT/Paren + if predicate.find(exp.Subquery) is not None: + return "fallback" + if predicate.find(exp.Select) is not None: + return "fallback" + if predicate.find(exp.AggFunc) is not None: + return "fallback" + if predicate.find(exp.Window) is not None: + return "fallback" + return "inline" + + @staticmethod + def _validate_extra_qualifiers( + extras: list[exp.Expression], + sides: "_IEJoinSides", + ) -> None: + """Raise for unqualified, unknown-aliased, or schema-qualified columns. + + Surfaces user mistakes at transpile time rather than deferring to a + downstream binder error. Three failure modes raise: + + - **Unqualified column:** no ``.table`` qualifier. + - **Unknown alias:** ``.table`` doesn't match either side of the join. + - **Catalog/schema-qualified column:** ``mycat.myschema.a.score`` + would silently mis-bind once the alias-rewriter remaps ``a`` to + the inner subquery's hardcoded ``a`` alias (the ``mycat.myschema`` + qualifier survives the rewrite and addresses a different relation + than intended). + """ + valid_aliases = {sides.left_user_alias, sides.right_user_alias} + for predicate in extras: + for col in predicate.find_all(exp.Column): + col_table = _normalize_alias(col.table) if col.table else "" + if not col_table: + raise _UnqualifiedProjectionError( + f"dialect='duckdb' cannot inline extra predicate " + f"{predicate.sql()!r}: column {col.sql()!r} must be " + f"qualified with {sides.left_user_alias!r} or " + f"{sides.right_user_alias!r}." + ) + if col_table not in valid_aliases: + raise _UnqualifiedProjectionError( + f"dialect='duckdb' cannot inline extra predicate " + f"{predicate.sql()!r}: column references unknown " + f"table qualifier {col.table!r}; expected " + f"{sides.left_user_alias!r} or " + f"{sides.right_user_alias!r}." + ) + if col.args.get("db") or col.args.get("catalog"): + raise _UnqualifiedProjectionError( + f"dialect='duckdb' cannot inline extra predicate " + f"{predicate.sql()!r}: column {col.sql()!r} carries " + f"a catalog/schema qualifier; use the alias-only " + f"form ({sides.left_user_alias!r}.or " + f"{sides.right_user_alias!r}.)." + ) + + @staticmethod + def _rewrite_refs_for_per_chrom_subquery( + node: exp.Expression, + sides: "_IEJoinSides", + ) -> str: + """Render a residual predicate against the inner subquery's ``a``/``b`` scope. + + The user's AST references their original aliases (``peaks a`` / + ``genes b`` or ``peaks p`` / ``genes g``), but the per-chromosome + inner subquery is emitted with hardcoded aliases ``a`` and ``b`` + regardless of what the user wrote. Rewrite each column reference + accordingly before emitting the residual SQL into the inner join's + ON clause. Alias matching is case-folded so mixed-case identifiers + match the DuckDB-equivalent contract. + + Assumes residuals have already been validated to reference only + ``sides.left_user_alias`` or ``sides.right_user_alias``; columns + without a table qualifier or pointing at an unknown alias must + have been rejected upstream. + """ + + def replace(n: exp.Expression) -> exp.Expression: + if not isinstance(n, exp.Column): + return n + col_table = _normalize_alias(n.table) if n.table else "" + if col_table == sides.left_user_alias: + new = n.copy() + new.set("table", exp.to_identifier(sides.left_inner_alias)) + return new + if col_table == sides.right_user_alias: + new = n.copy() + new.set("table", exp.to_identifier(sides.right_inner_alias)) + return new + return n + + rewritten = node.transform(replace, copy=True) + return rewritten.sql(dialect="duckdb") + + @staticmethod + def _rewrite_refs_for_wrapper_relation( + node: exp.Expression, + projection_map: dict[tuple[str, str], str], + sides: "_IEJoinSides", + clause_label: str, + ) -> str: + """Rewrite table-qualified column refs to the wrapper relation's inner alias. + + The dialect's outer ``SELECT`` reads from ``query(getvariable(...)) AS + ``. The wrapper relation's columns are whatever the inner + per-chromosome subqueries projected — i.e., the ``__giql_p`` names + on the left-hand side of each ``AS`` in the inner SELECT. Any user + ``ORDER BY`` / ``GROUP BY`` / ``HAVING`` (and any aggregate-function + argument) written against the original table aliases (``a.start``, + ``b.score``) must be rewritten to those inner alias names so DuckDB + can resolve them against the wrapper relation. Alias matching is + case-folded for DuckDB-equivalent semantics. + + An upstream pre-allocation step ensures every referenced column has + an entry in ``projection_map``; missing keys raise + :class:`_UnqualifiedProjectionError` as a guard against future scope + relaxations that would otherwise produce silent miscompiles. + + Scope caveat: this walker is not scope-aware — it rewrites every + :class:`~sqlglot.expressions.Column` whose ``.table`` matches either + side's alias anywhere inside ``node``, including correlated or + non-correlated subqueries that may rebind those aliases to a + different table. The dispatcher gates already reject any subquery + in GROUP BY / HAVING / ORDER BY / SELECT list so the unsafe shape + can't reach here today. A future relaxation of those gates must + add a scope-aware walker here. + """ + valid_aliases = (sides.left_user_alias, sides.right_user_alias) + + def replace(n: exp.Expression) -> exp.Expression: + if not isinstance(n, exp.Column): + return n + col_table = _normalize_alias(n.table) if n.table else "" + if col_table not in valid_aliases: + return n + key = (col_table, n.name) + if key not in projection_map: + raise _UnqualifiedProjectionError( + f"dialect='duckdb' cannot resolve {n.sql()!r} in " + f"{clause_label}: ensure the column is referenced " + f"with a table qualifier matching the join's left " + f"or right alias." + ) + return exp.column(projection_map[key]) + + rewritten = node.transform(replace, copy=True) + return rewritten.sql(dialect="duckdb") + + def _extract_extra_predicates( + self, + query: exp.Select, + intersects: Intersects, + ) -> list[exp.Expression]: + """Return the non-INTERSECTS predicate residuals from joins and WHERE. + + Walks the AND tree under each JOIN ON and the WHERE, strips the + target INTERSECTS node, and returns the residual predicates in + document order. An empty list means the query has no extras + beyond the target INTERSECTS. + + Note: :func:`_strip_intersects` only descends ``exp.And``. Predicates + whose AND tree wraps the INTERSECTS in ``exp.Or`` / ``exp.Not`` / + ``exp.Paren`` surface here with the INTERSECTS still embedded; + :meth:`_classify_extras` then routes them to the binned plan, while + :meth:`_validate_extra_qualifiers` enforces qualifier rules for the + remaining inlinable residuals. + """ + residuals: list[exp.Expression] = [] + for join in query.args.get("joins") or []: + on = join.args.get("on") + if on is None: + continue + residual = _strip_intersects(on, intersects) + if residual is not None: + residuals.append(residual) + where = query.args.get("where") + if where: + residual = _strip_intersects(where.this, intersects) + if residual is not None: + residuals.append(residual) + return residuals + + def _find_target_join( + self, query: exp.Select + ) -> tuple[Intersects | None, exp.Join | None]: + """Locate the single INTERSECTS join target. + + Returns ``(intersects_node, join)`` where ``join`` is the + :class:`~sqlglot.expressions.Join` AST node carrying the target + INTERSECTS — either directly via its ``ON`` clause, or via an + implicit cross-join whose ``INTERSECTS`` predicate lives in the + top-level ``WHERE``. Alias matching on the WHERE-branch is + case-folded for DuckDB-equivalent semantics. + """ + for join in query.args.get("joins") or []: + on = join.args.get("on") + if on: + intersects = _find_column_intersects_in(on) + if intersects: + if not isinstance(join.this, exp.Table): + return None, None + return intersects, join + + where = query.args.get("where") + if where: + intersects = _find_column_intersects_in(where.this) + if intersects: + from_table = query.args["from_"].this + if not isinstance(from_table, exp.Table): + return None, None + from_alias = _normalize_alias(from_table.alias_or_name) + left_alias = _normalize_alias(intersects.this.table) + right_alias = _normalize_alias(intersects.expression.table) + target_alias = right_alias if left_alias == from_alias else left_alias + for join in query.args.get("joins") or []: + if isinstance(join.this, exp.Table): + if _normalize_alias(join.this.alias_or_name) == target_alias: + return intersects, join + + return None, None + + def _build_sql( + self, + query: exp.Select, + intersects: Intersects, + sides: "_IEJoinSides", + the_join: exp.Join, + ) -> str | None: + """Render the multi-statement DuckDB IEJoin SQL string for *query*. + + Returns ``None`` when a soft-fallback condition fires (the residual + of the join carries a shape the binned plan can handle but the + dialect cannot inline, or a single-column USING references a column + that is not both tables' ``chrom_col``). Raises + :class:`_UnqualifiedProjectionError` when a user-mistake condition + fires; callers translating to the public surface wrap the raise to + :class:`ValueError`. + """ + extras = self._extract_extra_predicates(query, intersects) + if extras and self._classify_extras(extras) == "fallback": + return None + if extras: + self._validate_extra_qualifiers(extras, sides) + + # Tables registry is keyed by the bare table name; an unregistered + # table uses default column names like the binned plan does. + l_table = self.tables.get(sides.left_table.name) + r_table = self.tables.get(sides.right_table.name) + l_chrom = l_table.chrom_col if l_table else DEFAULT_CHROM_COL + l_start = l_table.start_col if l_table else DEFAULT_START_COL + l_end = l_table.end_col if l_table else DEFAULT_END_COL + r_chrom = r_table.chrom_col if r_table else DEFAULT_CHROM_COL + r_start = r_table.start_col if r_table else DEFAULT_START_COL + r_end = r_table.end_col if r_table else DEFAULT_END_COL + + # USING() admission check (the gate already restricted to + # single-column USING). The dialect's per-chromosome partition IS + # the equi-join that USING() requests; if the USING + # column doesn't match both tables' chrom_col (or the chrom_cols + # diverge), fall back so the binned plan emits the proper + # equi-join. + using_cols = the_join.args.get("using") or [] + if using_cols: + using_name = _normalize_alias(using_cols[0].name) + if ( + _normalize_alias(l_chrom) != _normalize_alias(r_chrom) + or _normalize_alias(l_chrom) != using_name + ): + return None + + ( + inner_projections, + outer_projections, + projection_map, + ) = self._resolve_projections( + query, + sides, + l_table, + r_table, + (l_chrom, l_start, l_end), + (r_chrom, r_start, r_end), ) - join3_on = exp.And( - this=key_match, - expression=self._build_overlap( - other_alias, join_alias, other_cols, join_cols - ), + + # Per-call random token (full uuid4 hex = 128 bits) so the SET + # VARIABLE name is collision-resistant even across many transpile() + # calls interleaved in one DuckDB session. DuckDB session variables + # are global session state, so token collision would silently + # rebind the wrapper query to a different intersection. + token = uuid4().hex + var_name = f"__giql_iejoin_{token}" + + # Canonicalize endpoints to 0-based half-open and use strict + # operators. Mixing inclusive operators with raw closed-closed + # coordinates would silently report non-overlapping touching + # intervals as overlapping. + q = self._sql_quote_ident + l_start_expr = canonical_start(f"a.{q(l_start)}", l_table) + l_end_expr = canonical_end(f"a.{q(l_end)}", l_table) + r_start_expr = canonical_start(f"b.{q(r_start)}", r_table) + r_end_expr = canonical_end(f"b.{q(r_end)}", r_table) + + # Render each table operand qualified (catalog/db/name) without the + # user's alias so the empty-schema fallback can supply its own + # ``a``/``b`` alias without producing ``FROM peaks AS a a`` shapes. + l_table_ident = self._qualified_table_ident(sides.left_table) + r_table_ident = self._qualified_table_ident(sides.right_table) + + # Join keyword switches by ``sides.kind``: SEMI / ANTI engage + # DuckDB's IE_JOIN with the corresponding semi/anti semantics; INNER + # is the default. (CROSS folded to INNER inside ``_IEJoinSides``.) + join_keyword = { + "INNER": "JOIN", + "SEMI": "SEMI JOIN", + "ANTI": "ANTI JOIN", + }[sides.kind] + + inner_select_list = ", ".join(inner_projections) + per_chrom_select_prefix = f"SELECT {inner_select_list} " + from_left = f"FROM (SELECT * FROM {l_table_ident} WHERE {q(l_chrom)} = " + from_right_open = ( + f") a {join_keyword} (SELECT * FROM {r_table_ident} WHERE {q(r_chrom)} = " ) - if extra: - join3_on = exp.And(this=join3_on, expression=extra) - join3_kwargs: dict = { - "this": exp.Table( - this=exp.Identifier(this=join_table_name), - alias=exp.TableAlias(this=exp.Identifier(this=join_alias)), - ), - "on": join3_on, - } - if join_side: - join3_kwargs["side"] = join_side + on_clause_predicates = [ + f"{l_start_expr} < {r_end_expr}", + f"{l_end_expr} > {r_start_expr}", + ] + for residual in extras: + on_clause_predicates.append( + self._rewrite_refs_for_per_chrom_subquery(residual, sides) + ) + on_clause = ") b ON " + " AND ".join(on_clause_predicates) + + # The dynamic SQL builder, expressed as a SQL string-concat that + # ``string_agg`` aggregates per-chromosome. Single quotes are + # doubled because the result is itself an SQL string literal that + # contains SQL. + chrom_literal = "'''' || replace(chrom, '''', '''''') || ''''" + + esc = self._sql_escape + per_chrom_sql_expr = ( + f"'{esc(per_chrom_select_prefix)}{esc(from_left)}' " + f"|| {chrom_literal} " + f"|| '{esc(from_right_open)}' " + f"|| {chrom_literal} " + f"|| '{esc(on_clause)}'" + ) + + # Empty-schema fallback when no chromosomes pass the partition + # filter. Project from the source tables under WHERE FALSE so + # DuckDB resolves every output column to its real declared type. + # SEMI / ANTI outputs reference only the left side, so we drop + # the right operand from the empty-schema FROM clause. + empty_select_list = ", ".join(inner_projections) + if sides.is_left_only_join: + empty_schema = ( + f"SELECT {empty_select_list} FROM {l_table_ident} a WHERE FALSE" + ) + else: + empty_schema = ( + f"SELECT {empty_select_list} " + f"FROM {l_table_ident} a, {r_table_ident} b WHERE FALSE" + ) + + # Partition source: INTERSECT for INNER / SEMI (a chromosome only + # on one side produces no matches anyway), left-distinct for ANTI + # (a chromosome present only on the left must still emit its rows + # under ANTI semantics). + if sides.kind == "ANTI": + chrom_partition_subquery = ( + f"SELECT DISTINCT {q(l_chrom)} AS chrom FROM {l_table_ident}" + ) + else: + chrom_partition_subquery = ( + f"SELECT DISTINCT {q(l_chrom)} AS chrom FROM {l_table_ident} " + f"INTERSECT SELECT DISTINCT {q(r_chrom)} AS chrom " + f"FROM {r_table_ident}" + ) + + set_var_stmt = ( + f"SET VARIABLE {var_name} = COALESCE((\n" + f" SELECT string_agg(\n" + f" {per_chrom_sql_expr},\n" + f" ' UNION ALL '\n" + f" )\n" + f" FROM ({chrom_partition_subquery})\n" + f"), '{esc(empty_schema)}')" + ) + + # Constant wrapper alias — uniqueness is provided by the + # token-bearing session variable; the outer SELECT's relation + # alias is never user-visible and doesn't need to vary per call. + wrapper_alias = "__giql_iejoin_wrapper" + outer_projection = ", ".join(outer_projections) + distinct_kw = "DISTINCT " if query.args.get("distinct") else "" + outer_select_parts = [ + f"SELECT {distinct_kw}{outer_projection} " + f"FROM query(getvariable('{var_name}')) AS {wrapper_alias}" + ] + + # The rewriter translates any ``a.col`` / ``b.col`` references in + # the GROUP BY / HAVING / ORDER BY clauses to the wrapper + # relation's inner-alias column names (``__giql_p``), since + # the user's original aliases are not in scope in the outer SELECT. + group_node = query.args.get("group") + if group_node is not None: + outer_select_parts.append( + self._rewrite_refs_for_wrapper_relation( + group_node, projection_map, sides, "GROUP BY" + ) + ) + + having_node = query.args.get("having") + if having_node is not None: + outer_select_parts.append( + self._rewrite_refs_for_wrapper_relation( + having_node, projection_map, sides, "HAVING" + ) + ) + + order_node = query.args.get("order") + if order_node is not None: + outer_select_parts.append( + self._rewrite_refs_for_wrapper_relation( + order_node, projection_map, sides, "ORDER BY" + ) + ) - join3 = exp.Join(**join3_kwargs) + limit_node = query.args.get("limit") + if limit_node is not None: + outer_select_parts.append(limit_node.sql(dialect="duckdb")) + + offset_node = query.args.get("offset") + if offset_node is not None: + outer_select_parts.append(offset_node.sql(dialect="duckdb")) + + outer_select = " ".join(outer_select_parts) + + return set_var_stmt + ";\n" + outer_select + + def _resolve_projections( + self, + query: exp.Select, + sides: "_IEJoinSides", + l_table: Table | None, + r_table: Table | None, + l_cols: tuple[str, str, str], + r_cols: tuple[str, str, str], + ) -> tuple[list[str], list[str], dict[tuple[str, str], str]]: + """Resolve the SELECT list (and any modifier-referenced columns). + + Returns ``(inner_projections, outer_projections, projection_map)``: + + * ``inner_projections`` is the per-chromosome subquery's SELECT + list — every column the query touches in any clause, projected + once each under a unique ``__giql_p`` alias. + * ``outer_projections`` is the outer SELECT's projection list, + rendered using the inner aliases (qualified columns, aggregate + calls, ``a.*``/``b.*`` expansions including ``strand_col`` when + the corresponding :class:`~giql.table.Table` config declares one). + * ``projection_map`` binds each ``(normalized_alias, column_name)`` + referenced anywhere in the query to its inner alias name so the + wrapper-relation rewriter and aggregate-argument rewriting can + translate user-written references to the wrapper relation. + + Raises :class:`_UnqualifiedProjectionError` for SELECT-list shapes + the dialect cannot translate (bare ``*``, unqualified column, + unknown table qualifier, ``a.* AS x``, window aggregate, ``FILTER`` + clause, scalar subquery, arithmetic over an aggregate, aggregate + over an unqualified column), and (when ``sides.is_left_only_join``) + any reference to the right side of the join. + """ + l_chrom, l_start, l_end = l_cols + r_chrom, r_start, r_end = r_cols + q = self._sql_quote_ident + l_alias = sides.left_user_alias + r_alias = sides.right_user_alias + + def star_columns( + table: Table | None, defaults: tuple[str, str, str] + ) -> tuple[str, ...]: + """Expand ``a.*`` / ``b.*`` to the genomic columns declared by + the :class:`~giql.table.Table` config (chrom / start / end, plus + ``strand_col`` when set). Without this, ``strand_col`` would + silently disappear under ``dialect='duckdb'`` even when the + user has registered it. + """ + chrom, start, end = defaults + if table is not None and table.strand_col: + return (chrom, start, end, table.strand_col) + return (chrom, start, end) + + inner_projections: list[str] = [] + projection_map: dict[tuple[str, str], str] = {} + + def reject_right_side_if_left_only(tbl_normalized: str, source_sql: str) -> None: + """Raise when a SEMI / ANTI join projects from the right side.""" + if sides.is_left_only_join and tbl_normalized == r_alias: + raise _UnqualifiedProjectionError( + f"dialect='duckdb' with kind={sides.kind!r} only " + f"projects left-side columns; got {source_sql!r} " + f"which references the right side " + f"({sides.right_user_alias!r})." + ) + + def allocate(tbl: str, col: str) -> str: + """Return the inner alias for ``(tbl, col)``, allocating if new. + + Allocation order is deterministic per query but depends on + (a) the fixed pre-walk order in :meth:`_resolve_projections` — + modifier columns first (``("group", "having", "order")``), + then aggregate-argument columns, then the SELECT list — and + (b) sqlglot's ``find_all`` traversal order within each + sub-clause. Tests therefore should NOT couple to a specific + ``__giql_p`` ↔ source-column mapping (e.g. + ``assert projection_map[(a, start)] == "__giql_p3"``). + """ + tbl = _normalize_alias(tbl) + key = (tbl, col) + existing = projection_map.get(key) + if existing is not None: + return existing + if tbl == l_alias: + side = sides.left_inner_alias + elif tbl == r_alias: + side = sides.right_inner_alias + else: + raise _UnqualifiedProjectionError( + f"Unknown table qualifier {tbl!r} in projection; " + f"expected {l_alias!r} or {r_alias!r}." + ) + alias = f"__giql_p{len(inner_projections)}" + inner_projections.append(f"{side}.{q(col)} AS {alias}") + projection_map[key] = alias + return alias + + def render_aggregate(agg_node: exp.Expression, user_alias: str | None) -> None: + """Validate aggregate-argument qualifiers and emit the outer projection.""" + for col in agg_node.find_all(exp.Column): + col_table = _normalize_alias(col.table) if col.table else "" + if col_table not in (l_alias, r_alias): + raise _UnqualifiedProjectionError( + "dialect='duckdb' requires qualified aggregate " + f"arguments; got {agg_node.sql()!r}. Use a " + "qualified column reference like " + f"'{type(agg_node).__name__.upper()}(a.col)' or " + f"'{type(agg_node).__name__.upper()}(b.col)'." + ) + reject_right_side_if_left_only(col_table, agg_node.sql()) + rewritten_agg = self._rewrite_refs_for_wrapper_relation( + agg_node, projection_map, sides, "aggregate argument" + ) + outer_name = ( + user_alias + if user_alias is not None + else f"__giql_agg_{len(outer_projections)}" + ) + outer_projections.append(f"{rewritten_agg} AS {q(outer_name)}") + + # Pre-allocate inner aliases for every column referenced in + # GROUP BY / HAVING / ORDER BY so the wrapper relation exposes + # them even if the user didn't put them in the SELECT list. + for clause_key in ("group", "having", "order"): + modifier_node = query.args.get(clause_key) + if modifier_node is None: + continue + for col in modifier_node.find_all(exp.Column): + col_table = _normalize_alias(col.table) if col.table else "" + if col_table in (l_alias, r_alias): + reject_right_side_if_left_only(col_table, col.sql()) + allocate(col_table, col.name) + + # Pre-allocate inner aliases for columns referenced inside any + # aggregate function in the SELECT list (we'll rewrite the + # aggregate's argument to use the inner alias when rendering the + # outer projection below). Peel any ``exp.Paren`` wrapper first + # so a paren-wrapped aggregate like ``(SUM(a.score)) AS s`` is + # treated identically to its bare form. + for sel in query.expressions: + target = sel.this if isinstance(sel, exp.Alias) else sel + while isinstance(target, exp.Paren): + target = target.this + if isinstance(target, exp.AggFunc): + for col in target.find_all(exp.Column): + col_table = _normalize_alias(col.table) if col.table else "" + if col_table in (l_alias, r_alias): + reject_right_side_if_left_only(col_table, col.sql()) + allocate(col_table, col.name) + + outer_projections: list[str] = [] + + for sel in query.expressions: + target = sel.this if isinstance(sel, exp.Alias) else sel + user_alias = sel.alias if isinstance(sel, exp.Alias) else None + # Peel paren wrappers at the top of dispatch so paren-wrapped + # aggregates like ``(SUM(a.score)) AS s`` route through the + # AggFunc branch below, and paren-wrapped diagnostic shapes + # (``(SUM(a.score) OVER (...))``) hit the targeted error + # branches below rather than the generic catch-all. + while isinstance(target, exp.Paren): + target = target.this + + if isinstance(target, exp.Star) and not isinstance(sel, exp.Column): + raise _UnqualifiedProjectionError( + "dialect='duckdb' requires qualified projections; bare '*' " + "is not supported. Use 'a.*', 'b.*', or qualified columns." + ) + + if isinstance(target, exp.Column) and isinstance(target.this, exp.Star): + # Reject ``a.* AS x`` — most SQL engines reject + # star-with-alias outright, and the dialect has no + # reasonable interpretation (apply ``x`` as a prefix? + # as a single composite column name?). + if user_alias is not None: + raise _UnqualifiedProjectionError( + "dialect='duckdb' does not support star projections " + f"with a user alias; got {sel.sql()!r}. Either drop " + "the alias (use ``a.*``) or list the columns " + "explicitly with per-column aliases." + ) + tbl = _normalize_alias(target.table) if target.table else "" + if tbl == l_alias: + cols = star_columns(l_table, (l_chrom, l_start, l_end)) + elif tbl == r_alias: + cols = star_columns(r_table, (r_chrom, r_start, r_end)) + else: + raise _UnqualifiedProjectionError( + f"Unknown table qualifier {target.table!r} in projection; " + f"expected {l_alias!r} or {r_alias!r}." + ) + reject_right_side_if_left_only(tbl, sel.sql()) + # Expand to the genomic columns the Table config knows + # about; arbitrary additional columns require explicit + # listing since schema introspection isn't available at + # transpile time. + for col in cols: + alias = allocate(tbl, col) + outer_projections.append(f"{alias} AS {q(col)}") + continue + + if isinstance(target, exp.AggFunc): + render_aggregate(target, user_alias) + continue + + if isinstance(target, exp.Column) and target.table: + tbl = _normalize_alias(target.table) + reject_right_side_if_left_only(tbl, target.sql()) + col = target.name + alias = allocate(tbl, col) + outer_name = user_alias if user_alias is not None else col + outer_projections.append(f"{alias} AS {q(outer_name)}") + continue + + # Diagnostic branches for shapes the dialect deliberately + # rejects. Specialize the error message when we can recognize + # the unsupported shape so the user isn't told "use a + # qualified column" when they actually wrote an aggregate + # variant the dispatcher couldn't classify. Direct-isinstance + # branches catch bare shapes; the ``target.find(...)`` branches + # below catch nested forms (``CAST(SUM(a.x) OVER (...) AS + # DOUBLE)`` etc.) before they fall through to the generic + # AggFunc-found catch-all. + if isinstance(target, exp.Window): + raise _UnqualifiedProjectionError( + "dialect='duckdb' does not support window aggregates " + f"in the SELECT list; got {target.sql()!r}. Either " + "drop the OVER (...) clause, or omit dialect='duckdb' " + "to use the binned plan." + ) + if isinstance(target, exp.Filter): + raise _UnqualifiedProjectionError( + "dialect='duckdb' does not support FILTER (WHERE ...) " + f"clauses on aggregates; got {target.sql()!r}. Either " + "rewrite the FILTER as a WHERE predicate alongside " + "the INTERSECTS, or omit dialect='duckdb'." + ) + if isinstance(target, exp.Subquery): + # A scalar subquery in the SELECT list (e.g. + # ``(SELECT SUM(x) FROM other_table)``) is shaped like an + # aggregate but doesn't fit the dialect's projection rules. + raise _UnqualifiedProjectionError( + "dialect='duckdb' does not support scalar subqueries in " + f"the SELECT list; got {target.sql()!r}. Either rewrite " + "without the subquery, or omit dialect='duckdb' to use " + "the binned plan." + ) + if target.find(exp.Window) is not None: + raise _UnqualifiedProjectionError( + "dialect='duckdb' does not support window aggregates " + f"in the SELECT list; got {target.sql()!r}. Either " + "drop the OVER (...) clause, or omit dialect='duckdb' " + "to use the binned plan." + ) + if target.find(exp.Filter) is not None: + raise _UnqualifiedProjectionError( + "dialect='duckdb' does not support FILTER (WHERE ...) " + f"clauses on aggregates; got {target.sql()!r}. Either " + "rewrite the FILTER as a WHERE predicate alongside " + "the INTERSECTS, or omit dialect='duckdb'." + ) + if target.find(exp.AggFunc) is not None: + raise _UnqualifiedProjectionError( + "dialect='duckdb' does not support aggregates inside " + f"arithmetic or function expressions; got {target.sql()!r}. " + "Project the underlying aggregate as its own column and " + "do the arithmetic in a wrapping query, or omit " + "dialect='duckdb'." + ) + raise _UnqualifiedProjectionError( + "dialect='duckdb' requires qualified projections; got " + f"{target.sql()!r}. Use a qualified column reference like " + "'a.col' or 'b.col [AS x]'." + ) + + if not outer_projections: + raise _UnqualifiedProjectionError( + "dialect='duckdb' requires at least one qualified projection." + ) - return [join1, join2, join3] + # An aggregate without any source-column argument (e.g. + # ``COUNT(*)``) leaves ``inner_projections`` empty in queries that + # have no other column references. The per-chromosome subquery + # still needs at least one column to project, so emit a constant + # placeholder that DuckDB optimizes away. (Without this the + # generated SQL ``SELECT FROM (...) a JOIN (...) b ON ...`` would + # fail to parse.) + if not inner_projections: + inner_projections.append("1 AS __giql_placeholder") + + return inner_projections, outer_projections, projection_map From c2204d42f04df6020703f6be04e144be003cd54b Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:03:01 -0400 Subject: [PATCH 028/142] feat: Expose dialect kwarg on transpile() with mutual-exclusivity guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit transpile() gains a dialect: Literal["duckdb"] | None = None keyword. Two @overload declarations encode the surface: one for the binned path (dialect: None, intersects_bin_size: int | None) and one for the dialect path (dialect: Literal["duckdb"], intersects_bin_size: None). The implementation guards the mutual exclusivity at runtime with a ValueError whose message points the caller at the right knob to drop, and an unknown dialect value echoes the offending value back to the caller. When dialect="duckdb" is set, transpile() first asks IntersectsDuckDBIEJoinTransformer whether the query engages the dialect path. If so, the transformer returns a complete SQL string (SET VARIABLE + SELECT through query(getvariable(...))) which is returned verbatim. If the transformer declines (None), the binned plan runs as before — so callers can opt in unconditionally without hitting silent correctness regressions on unsupported shapes. A _reraise_as_value_error context manager surrounds the parse, transformer chain, and generator stages. Existing ValueError raises propagate verbatim (so the dialect's targeted error messages survive the boundary); unexpected exceptions wrap to ValueError with a stage-prefixed message and, for parse errors, the original input appended so the offending text is preserved. Default transpile() behavior is unchanged: when dialect is None, the existing binned plan runs. --- src/giql/transpile.py | 176 ++++++++++++++++++++++++++++++------------ 1 file changed, 126 insertions(+), 50 deletions(-) diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 94eea2a..98add46 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -4,6 +4,11 @@ to standard SQL. """ +from contextlib import contextmanager +from typing import Iterator +from typing import Literal +from typing import overload + from sqlglot import parse_one from giql.dialect import GIQLDialect @@ -12,41 +17,35 @@ from giql.table import Tables from giql.transformer import ClusterTransformer from giql.transformer import IntersectsBinnedJoinTransformer +from giql.transformer import IntersectsDuckDBIEJoinTransformer from giql.transformer import MergeTransformer -def _build_tables(tables: list[str | Table] | None) -> Tables: - """Build a Tables container from table specifications. - - Parameters - ---------- - tables : list[str | Table] | None - Table specifications. Strings use default column mappings. - Table objects provide custom column mappings. - - Returns - ------- - Tables - Container with all tables registered. - """ - container = Tables() +@overload +def transpile( + giql: str, + tables: list[str | Table] | None = None, + *, + dialect: None = None, + intersects_bin_size: int | None = None, +) -> str: ... - if tables is None: - return container - for item in tables: - if isinstance(item, str): - container.register(item, Table(item)) - else: - container.register(item.name, item) - - return container +@overload +def transpile( + giql: str, + tables: list[str | Table] | None = None, + *, + dialect: Literal["duckdb"], + intersects_bin_size: None = None, +) -> str: ... def transpile( giql: str, tables: list[str | Table] | None = None, *, + dialect: Literal["duckdb"] | None = None, intersects_bin_size: int | None = None, ) -> str: """Transpile a GIQL query to SQL. @@ -59,15 +58,26 @@ def transpile( giql : str The GIQL query string containing genomic extensions like INTERSECTS, CONTAINS, WITHIN, CLUSTER, MERGE, NEAREST, or DISJOIN. - tables : list[str | Table] | None + tables : list[str | :class:`Table`] | None Table configurations. Strings use default column mappings - (chrom, start, end, strand). Table objects provide custom - column name mappings. + (chrom, start, end, strand). :class:`Table` objects provide + custom column name mappings. + dialect : Literal["duckdb"] | None + Optional target dialect. When set to ``"duckdb"``, column-to-column + ``INTERSECTS`` joins (INNER, SEMI, or ANTI) are transpiled into a + per-chromosome dynamic-SQL pattern (``SET VARIABLE`` + + ``query(getvariable(...))``) that DuckDB plans through its + range-join family (``IE_JOIN`` / ``PIECEWISE_MERGE_JOIN``). + Mutually exclusive with ``intersects_bin_size``. Defaults to + ``None`` (the generic binned equi-join path). Hard-error projection + shapes raise ``ValueError`` at transpile time; see the performance + guide for the full enumeration. intersects_bin_size : int | None Bin size for INTERSECTS equi-join optimization. When a query contains a full-table column-to-column INTERSECTS join, the transpiler rewrites it as a binned equi-join for performance. - Defaults to 10,000 if not specified. + Defaults to 10,000 if not specified. Cannot be combined with + ``dialect="duckdb"``. Returns ------- @@ -77,7 +87,9 @@ def transpile( Raises ------ ValueError - If the query cannot be parsed or transpiled. + If the query cannot be parsed or transpiled, if ``dialect`` is + unknown, or if ``dialect="duckdb"`` and ``intersects_bin_size`` + are both set. Examples -------- @@ -88,7 +100,7 @@ def transpile( tables=["peaks"], ) - Custom table configuration:: + Custom :class:`Table` configuration:: sql = transpile( "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'", @@ -111,41 +123,105 @@ def transpile( tables=["peaks", "genes"], intersects_bin_size=100000, ) + + DuckDB IEJoin dialect (column-to-column INNER/SEMI/ANTI JOIN only; + projections must be qualified):: + + sql = transpile( + "SELECT a.chrom, a.start, b.start " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) """ - # Build tables container + if dialect is not None and dialect != "duckdb": + raise ValueError(f"Unknown dialect: {dialect!r}. Supported: 'duckdb' or None.") + if dialect == "duckdb" and intersects_bin_size is not None: + raise ValueError( + "intersects_bin_size has no effect with dialect='duckdb'; " + "the DuckDB dialect uses an IEJoin per-partition plan instead " + "of the binned equi-join. Pass one or the other, not both." + ) + tables_container = _build_tables(tables) - # Initialize transformers with table configurations + with _reraise_as_value_error("Parse error", query=giql): + ast = parse_one(giql, dialect=GIQLDialect) + + # Falls back to the binned plan for unsupported shapes — see + # IntersectsDuckDBIEJoinTransformer.transform_to_sql for the complete + # fallback set. + if dialect == "duckdb": + duckdb_transformer = IntersectsDuckDBIEJoinTransformer(tables_container) + with _reraise_as_value_error("Transformation error"): + duckdb_sql = duckdb_transformer.transform_to_sql(ast) + if duckdb_sql is not None: + return duckdb_sql + intersects_transformer = IntersectsBinnedJoinTransformer( tables_container, bin_size=intersects_bin_size, ) merge_transformer = MergeTransformer(tables_container) cluster_transformer = ClusterTransformer(tables_container) - - # Initialize generator with table configurations generator = BaseGIQLGenerator(tables=tables_container) - # Parse GIQL query - try: - ast = parse_one(giql, dialect=GIQLDialect) - except Exception as e: - raise ValueError(f"Parse error: {e}\nQuery: {giql}") from e - - # Apply transformations - try: + with _reraise_as_value_error("Transformation error"): ast = intersects_transformer.transform(ast) - # MERGE transformation (which may internally use CLUSTER) ast = merge_transformer.transform(ast) - # CLUSTER transformation for any standalone CLUSTER expressions ast = cluster_transformer.transform(ast) - except Exception as e: - raise ValueError(f"Transformation error: {e}") from e - # Generate SQL - try: + with _reraise_as_value_error("Transpilation error"): sql = generator.generate(ast) - except Exception as e: - raise ValueError(f"Transpilation error: {e}") from e return sql + + +def _build_tables(tables: list[str | Table] | None) -> Tables: + """Build a :class:`Tables` container from table specifications. + + Parameters + ---------- + tables : list[str | :class:`Table`] | None + Table specifications. Strings use default column mappings. + :class:`Table` objects provide custom column mappings. + + Returns + ------- + Tables + Container with all tables registered. + """ + container = Tables() + + if tables is None: + return container + + for item in tables: + if isinstance(item, str): + container.register(item, Table(item)) + else: + container.register(item.name, item) + + return container + + +@contextmanager +def _reraise_as_value_error(prefix: str, query: str | None = None) -> Iterator[None]: + """Re-raise non-:class:`ValueError` exceptions as :class:`ValueError` with *prefix*. + + Lets user-facing :class:`ValueError`\\s from the parser, transformer chain, + and generator propagate verbatim (so the dialect's targeted error messages + survive the boundary) while wrapping unexpected exceptions in a uniform + :class:`ValueError` prefixed with the stage name. When *query* is supplied, + the original input is appended to the message so parse errors retain the + offending text. + """ + try: + yield + except ValueError: + raise + except Exception as e: + msg = f"{prefix}: {e}" + if query is not None: + msg += f"\nQuery: {query}" + raise ValueError(msg) from e From b420de6c1236a9113a03bbdbf229c8d67a39bbd9 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:03:16 -0400 Subject: [PATCH 029/142] test: Add DuckDB IEJoin dialect test coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 157 tests across two new files: tests/test_canonical.py — 19 tests covering canonical_start, canonical_end, decanonical_start, and decanonical_end as module-level free functions. Example-based tests pin every (coordinate_system, interval_type) combination; three Hypothesis property tests verify the documented-form invariant on canonical_end and the round-trip inverse property on decanonical_start / decanonical_end across the full enum cross-product. tests/test_duckdb_iejoin.py — 138 tests in three classes: - TestTranspileDuckDBIEJoinSQLStructure asserts SQL shape and routing: which queries engage the dialect, which fall back, which raise. Covers supported decorations (custom column names, literal-range pass-through, qualified-star expansion, aggregates, GROUP BY / HAVING / ORDER BY / LIMIT / OFFSET / DISTINCT / extra-predicate inlining including BETWEEN and IS NULL), every documented fallback trigger, and every ValueError raise path. - TestTranspileDuckDBIEJoinExecution runs end-to-end against in- memory DuckDB: multi-chromosome correctness, all four (coordinate_system, interval_type) combinations including mixed, empty-input cases, typed empty-schema fallback, chromosome-literal escaping, custom column names, a.* / b.* expansion, GROUP BY / HAVING aggregates including COUNT(DISTINCT) and AVG, ORDER BY DESC across chromosomes, multi-predicate inlining, NULL-aware predicates, user-aliased joins with cross-side predicates, ambiguous-outer-name resolution, paren-wrapped aggregates, and a kitchen-sink query. Hypothesis property tests cross-check the dialect against a Python- native overlap reference, against the binned plan over random inputs (including a full Tier 1 stack with LIMIT 0, OFFSET, and empty-input cases), and across arbitrary printable-ASCII chromosome names. A parametrize test pins cross-plan equivalence across all four (coordinate_system, interval_type) combinations. - TestTranspileDuckDBIEJoinKwargs covers the @overload pair, the dialect-vs-intersects_bin_size mutex with both kwargs named in the error, the unknown-dialect echo, and one cross-kwarg behavioral regression confirming intersects_bin_size still works when dialect is omitted. --- tests/test_canonical.py | 512 ++++ tests/test_duckdb_iejoin.py | 5580 +++++++++++++++++++++++++++++++++++ 2 files changed, 6092 insertions(+) create mode 100644 tests/test_canonical.py create mode 100644 tests/test_duckdb_iejoin.py diff --git a/tests/test_canonical.py b/tests/test_canonical.py new file mode 100644 index 0000000..93c1c62 --- /dev/null +++ b/tests/test_canonical.py @@ -0,0 +1,512 @@ +"""Unit tests for ``giql.canonical`` coordinate canonicalization helpers. + +These tests pin down the externally observable contract of +:func:`giql.canonical.canonical_start` / :func:`giql.canonical.canonical_end` +and their inverses :func:`giql.canonical.decanonical_start` / +:func:`giql.canonical.decanonical_end` — pure functions that wrap a raw +column reference with the offset required to convert between a source +:class:`giql.table.Table`'s declared encoding and the canonical 0-based +half-open form the transpiler emits. +""" + +import pytest + +from giql.canonical import canonical_end +from giql.canonical import canonical_start +from giql.canonical import decanonical_end +from giql.canonical import decanonical_start +from giql.table import Table + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import given # noqa: E402 +from hypothesis import settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + + +def test_canonical_start_should_return_input_unchanged_when_table_is_none(): + """Test that a missing table leaves the start expression untouched. + + Given: + A raw start expression and ``table=None``. + When: + ``canonical_start(raw, None)`` is called. + Then: + It should return the original raw expression unchanged. + """ + # Arrange + raw = "a.start" + + # Act + result = canonical_start(raw, None) + + # Assert + assert result == raw + + +def test_canonical_start_should_return_input_unchanged_when_table_is_zero_based(): + """Test that a 0-based table leaves the start expression untouched. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="0based"``. + When: + ``canonical_start(raw, table)`` is called. + Then: + It should return ``raw`` unchanged because the source already + uses the canonical 0-based start convention. + """ + # Arrange + raw = "a.start" + table = Table(name="features", coordinate_system="0based") + + # Act + result = canonical_start(raw, table) + + # Assert + assert result == raw + + +def test_canonical_start_should_subtract_one_when_table_is_one_based(): + """Test that a 1-based table subtracts one from the start expression. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="1based"``. + When: + ``canonical_start(raw, table)`` is called. + Then: + It should return ``f"({raw} - 1)"`` so the canonical 0-based + half-open form is emitted downstream. + """ + # Arrange + raw = "a.start" + table = Table(name="features", coordinate_system="1based") + + # Act + result = canonical_start(raw, table) + + # Assert + assert result == f"({raw} - 1)" + + +def test_canonical_end_should_return_input_unchanged_when_table_is_none(): + """Test that a missing table leaves the end expression untouched. + + Given: + A raw end expression and ``table=None``. + When: + ``canonical_end(raw, None)`` is called. + Then: + It should return the original raw expression unchanged. + """ + # Arrange + raw = "a.end" + + # Act + result = canonical_end(raw, None) + + # Assert + assert result == raw + + +def test_canonical_end_should_return_input_unchanged_when_table_is_zero_based_half_open(): + """Test that 0-based half-open tables leave the end expression untouched. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="0based"`` + and ``interval_type="half_open"``. + When: + ``canonical_end(raw, table)`` is called. + Then: + It should return ``raw`` unchanged because the source already + matches the canonical convention. + """ + # Arrange + raw = "a.end" + table = Table( + name="features", + coordinate_system="0based", + interval_type="half_open", + ) + + # Act + result = canonical_end(raw, table) + + # Assert + assert result == raw + + +def test_canonical_end_should_add_one_when_table_is_zero_based_closed(): + """Test that 0-based closed tables add one to the end expression. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="0based"`` + and ``interval_type="closed"``. + When: + ``canonical_end(raw, table)`` is called. + Then: + It should return ``f"({raw} + 1)"`` to convert closed to + half-open. + """ + # Arrange + raw = "a.end" + table = Table( + name="features", + coordinate_system="0based", + interval_type="closed", + ) + + # Act + result = canonical_end(raw, table) + + # Assert + assert result == f"({raw} + 1)" + + +def test_canonical_end_should_subtract_one_when_table_is_one_based_half_open(): + """Test that 1-based half-open tables subtract one from the end expression. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="1based"`` + and ``interval_type="half_open"``. + When: + ``canonical_end(raw, table)`` is called. + Then: + It should return ``f"({raw} - 1)"`` to convert 1-based + half-open to canonical 0-based half-open. + """ + # Arrange + raw = "a.end" + table = Table( + name="features", + coordinate_system="1based", + interval_type="half_open", + ) + + # Act + result = canonical_end(raw, table) + + # Assert + assert result == f"({raw} - 1)" + + +def test_canonical_end_should_return_input_unchanged_when_table_is_one_based_closed(): + """Test that 1-based closed tables leave the end expression untouched. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="1based"`` + and ``interval_type="closed"``. + When: + ``canonical_end(raw, table)`` is called. + Then: + It should return ``raw`` unchanged because a 1-based closed + end equals a 0-based half-open end numerically. + """ + # Arrange + raw = "a.end" + table = Table( + name="features", + coordinate_system="1based", + interval_type="closed", + ) + + # Act + result = canonical_end(raw, table) + + # Assert + assert result == raw + + +@settings(max_examples=100, deadline=None) +@given( + raw=st.text( + alphabet=st.characters(min_codepoint=0x20, max_codepoint=0x7E), + min_size=1, + max_size=20, + ), + coord_system=st.sampled_from(["0based", "1based"]), + interval_type=st.sampled_from(["half_open", "closed"]), +) +def test_canonical_end_should_return_one_of_three_documented_forms( + raw, coord_system, interval_type +): + """Test that canonical_end output matches one of three documented shapes. + + Given: + A Hypothesis-generated raw column reference (ASCII-printable + text of length 1..20) and a sampled + ``(coordinate_system, interval_type)`` enum pair drawn from + the four allowed combinations. + When: + ``canonical_end(raw, Table(...))`` is called. + Then: + It should return exactly one of ``raw``, ``f"({raw} + 1)"``, + or ``f"({raw} - 1)"`` — no other shape is ever produced. + """ + # Arrange + table = Table( + name="features", + coordinate_system=coord_system, + interval_type=interval_type, + ) + allowed = {raw, f"({raw} + 1)", f"({raw} - 1)"} + + # Act + result = canonical_end(raw, table) + + # Assert + assert result in allowed + + +def test_decanonical_start_should_return_input_unchanged_when_table_is_none(): + """Test that a missing table leaves the canonical start untouched. + + Given: + A canonical 0-based start expression and ``table=None``. + When: + ``decanonical_start(canon, None)`` is called. + Then: + It should return the canonical expression unchanged. + """ + # Arrange + canon = "a.start" + + # Act + result = decanonical_start(canon, None) + + # Assert + assert result == canon + + +def test_decanonical_start_should_return_input_unchanged_when_table_is_zero_based(): + """Test that a 0-based table leaves the canonical start untouched. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="0based"``. + When: + ``decanonical_start(canon, table)`` is called. + Then: + It should return ``canon`` unchanged because the canonical + form already matches the source convention. + """ + # Arrange + canon = "a.start" + table = Table(name="features", coordinate_system="0based") + + # Act + result = decanonical_start(canon, table) + + # Assert + assert result == canon + + +def test_decanonical_start_should_add_one_when_table_is_one_based(): + """Test that a 1-based table adds one to the canonical start. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="1based"``. + When: + ``decanonical_start(canon, table)`` is called. + Then: + It should return ``f"({canon} + 1)"`` so the value lands back + in the table's 1-based encoding. + """ + # Arrange + canon = "(a.start - 1)" + table = Table(name="features", coordinate_system="1based") + + # Act + result = decanonical_start(canon, table) + + # Assert + assert result == f"({canon} + 1)" + + +@settings(max_examples=100, deadline=None) +@given( + x=st.integers(min_value=0, max_value=10**9), + coord_system=st.sampled_from(["0based", "1based"]), +) +def test_decanonical_start_should_invert_canonical_start_numerically(x, coord_system): + """Test that decanonical_start is the numeric inverse of canonical_start. + + Given: + A Hypothesis-generated integer start coordinate and a sampled + ``coordinate_system``. + When: + ``decanonical_start(canonical_start(str(x), table), table)`` is + evaluated as a Python expression. + Then: + It should evaluate back to ``x`` — the two helpers are exact + inverses across the full coordinate-system enum. + """ + # Arrange + table = Table(name="features", coordinate_system=coord_system) + + # Act + composed = decanonical_start(canonical_start(str(x), table), table) + + # Assert + assert eval(composed) == x + + +def test_decanonical_end_should_return_input_unchanged_when_table_is_none(): + """Test that a missing table leaves the canonical end untouched. + + Given: + A canonical 0-based half-open end expression and ``table=None``. + When: + ``decanonical_end(canon, None)`` is called. + Then: + It should return the canonical expression unchanged. + """ + # Arrange + canon = "a.end" + + # Act + result = decanonical_end(canon, None) + + # Assert + assert result == canon + + +def test_decanonical_end_should_return_input_unchanged_when_table_is_zero_based_half_open(): + """Test that 0-based half-open tables leave the canonical end untouched. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="0based"`` + and ``interval_type="half_open"``. + When: + ``decanonical_end(canon, table)`` is called. + Then: + It should return ``canon`` unchanged because the canonical + form already matches the source convention. + """ + # Arrange + canon = "a.end" + table = Table( + name="features", + coordinate_system="0based", + interval_type="half_open", + ) + + # Act + result = decanonical_end(canon, table) + + # Assert + assert result == canon + + +def test_decanonical_end_should_subtract_one_when_table_is_zero_based_closed(): + """Test that 0-based closed tables subtract one from the canonical end. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="0based"`` + and ``interval_type="closed"``. + When: + ``decanonical_end(canon, table)`` is called. + Then: + It should return ``f"({canon} - 1)"`` to convert canonical + half-open back to the source's closed form. + """ + # Arrange + canon = "(a.end + 1)" + table = Table( + name="features", + coordinate_system="0based", + interval_type="closed", + ) + + # Act + result = decanonical_end(canon, table) + + # Assert + assert result == f"({canon} - 1)" + + +def test_decanonical_end_should_add_one_when_table_is_one_based_half_open(): + """Test that 1-based half-open tables add one to the canonical end. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="1based"`` + and ``interval_type="half_open"``. + When: + ``decanonical_end(canon, table)`` is called. + Then: + It should return ``f"({canon} + 1)"`` to convert canonical + half-open back to the source's 1-based half-open form. + """ + # Arrange + canon = "(a.end - 1)" + table = Table( + name="features", + coordinate_system="1based", + interval_type="half_open", + ) + + # Act + result = decanonical_end(canon, table) + + # Assert + assert result == f"({canon} + 1)" + + +def test_decanonical_end_should_return_input_unchanged_when_table_is_one_based_closed(): + """Test that 1-based closed tables leave the canonical end untouched. + + Given: + A :class:`giql.table.Table` with ``coordinate_system="1based"`` + and ``interval_type="closed"``. + When: + ``decanonical_end(canon, table)`` is called. + Then: + It should return ``canon`` unchanged because a 1-based closed + end equals a canonical 0-based half-open end numerically. + """ + # Arrange + canon = "a.end" + table = Table( + name="features", + coordinate_system="1based", + interval_type="closed", + ) + + # Act + result = decanonical_end(canon, table) + + # Assert + assert result == canon + + +@settings(max_examples=100, deadline=None) +@given( + x=st.integers(min_value=1, max_value=10**9), + coord_system=st.sampled_from(["0based", "1based"]), + interval_type=st.sampled_from(["half_open", "closed"]), +) +def test_decanonical_end_should_invert_canonical_end_numerically( + x, coord_system, interval_type +): + """Test that decanonical_end is the numeric inverse of canonical_end. + + Given: + A Hypothesis-generated integer end coordinate and a sampled + ``(coordinate_system, interval_type)`` pair drawn from the + four allowed combinations. + When: + ``decanonical_end(canonical_end(str(x), table), table)`` is + evaluated as a Python expression. + Then: + It should evaluate back to ``x`` — the two helpers are exact + inverses across the full enum cross-product. + """ + # Arrange + table = Table( + name="features", + coordinate_system=coord_system, + interval_type=interval_type, + ) + + # Act + composed = decanonical_end(canonical_end(str(x), table), table) + + # Assert + assert eval(composed) == x diff --git a/tests/test_duckdb_iejoin.py b/tests/test_duckdb_iejoin.py new file mode 100644 index 0000000..115eaf8 --- /dev/null +++ b/tests/test_duckdb_iejoin.py @@ -0,0 +1,5580 @@ +"""Tests for the DuckDB IEJoin dialect path on column-to-column INTERSECTS joins.""" + +import re + +import pytest + +from giql import Table +from giql import transpile + +duckdb = pytest.importorskip("duckdb") +hypothesis = pytest.importorskip("hypothesis") + +from hypothesis import HealthCheck # noqa: E402 +from hypothesis import given # noqa: E402 +from hypothesis import settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + + +def _make_table(conn, name: str, rows: list[tuple]) -> None: + conn.execute(f""" + CREATE TABLE {name} ( + chrom VARCHAR, + "start" INTEGER, + "end" INTEGER, + name VARCHAR, + score INTEGER, + strand VARCHAR + ) + """) + if rows: + conn.executemany(f"INSERT INTO {name} VALUES (?, ?, ?, ?, ?, ?)", rows) + + +def _python_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: + """Reference half-open overlap on (chrom, start, end) triples. + + Returns a *sorted multiset* (list) so the caller can assert exact row + multiplicity — a `set` would collapse duplicate input rows and hide + multiplicity bugs in the dialect's rewrite. + """ + out: list[tuple] = [] + for pc, ps, pe in peaks: + for gc, gs, ge in genes: + if pc == gc and pe > gs and ge > ps: + out.append((pc, ps, pe, gc, gs, ge)) + return sorted(out) + + +def _python_semi_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: + """Reference SEMI-join overlap: distinct left rows with any match.""" + matched: set[tuple] = set() + for pc, ps, pe in peaks: + for gc, gs, ge in genes: + if pc == gc and pe > gs and ge > ps: + matched.add((pc, ps, pe)) + break + return sorted(matched) + + +def _python_anti_overlap(peaks: list[tuple], genes: list[tuple]) -> list[tuple]: + """Reference ANTI-join overlap: distinct left rows with no match.""" + matched = set(_python_semi_overlap(peaks, genes)) + return sorted({(pc, ps, pe) for (pc, ps, pe) in peaks} - matched) + + +@pytest.fixture +def conn(): + c = duckdb.connect(":memory:") + yield c + c.close() + + +@pytest.fixture +def peaks_genes(conn): + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 10, "+"), + ("chr1", 300, 400, "p2", 20, "+"), + ("chr1", 500, 600, "p3", 25, "+"), + ("chr2", 100, 200, "p4", 30, "-"), + ("chr2", 800, 900, "p5", 35, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 500, 600, "g2", 2, "-"), + ("chr1", 700, 800, "g3", 3, "+"), + ("chr2", 50, 150, "g4", 4, "-"), + ("chr2", 250, 350, "g5", 5, "+"), + ], + ) + return conn + + +# Truth table for peaks_genes overlapping pairs (half-open): +# - chr1: peaks[100,200) vs genes[150,250) -> overlap +# - chr1: peaks[500,600) vs genes[500,600) -> overlap +# - chr2: peaks[100,200) vs genes[50,150) -> overlap +_EXPECTED_OVERLAPS_PEAKS_GENES = [ + ("chr1", 100, 200, "chr1", 150, 250), + ("chr1", 500, 600, "chr1", 500, 600), + ("chr2", 100, 200, "chr2", 50, 150), +] + + +class TestTranspileDuckDBIEJoinSQLStructure: + """SQL-shape and routing assertions for the DuckDB IEJoin dialect path.""" + + def test_transpile_should_keep_binned_path_when_dialect_is_none(self): + """Test that omitting ``dialect`` keeps the generic binned plan. + + Given: + A column-to-column INTERSECTS JOIN. + When: + ``transpile`` is called without specifying ``dialect``. + Then: + It should emit SQL that contains no ``SET VARIABLE`` / + ``getvariable`` scaffolding (the IEJoin path is opt-in). + """ + # Arrange + query = """ + SELECT a.*, b.* + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act + sql = transpile(query, tables=["peaks", "genes"]) + + # Assert + assert "SET VARIABLE" not in sql.upper() + assert "getvariable" not in sql + + def test_transpile_should_route_outer_join_intersects_to_binned_plan_when_dialect_is_duckdb( + self, conn + ): + """Test that LEFT JOIN INTERSECTS falls back to the binned plan. + + Given: + A LEFT JOIN INTERSECTS query and a peak with no overlapping + gene on the join chromosome. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should preserve LEFT JOIN semantics by returning the + unmatched peak with NULL gene columns (which the IEJoin path + cannot do, so the binned fallback fires). + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 0, "+"), + ("chr1", 5000, 6000, "p_lonely", 0, "+"), + ], + ) + _make_table(conn, "genes", [("chr1", 150, 250, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_start, b.start AS b_start + FROM peaks a + LEFT JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall(), key=lambda r: r[0]) + + # Assert + assert rows == [(100, 150), (5000, None)] + + def test_query_should_honor_extra_join_on_predicate_alongside_where_intersects_when_dialect_is_duckdb( + self, conn + ): + """Test that an extra non-INTERSECTS join predicate is honored. + + Given: + A query whose JOIN ON carries a non-INTERSECTS predicate + (``a.score > b.score``) alongside a WHERE INTERSECTS, and + input rows where one pair overlaps but violates the score + predicate. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The dialect path should inline the extra predicate into each + per-chromosome subquery's join ON, excluding the pair that + violates the predicate and including the pair that satisfies + it. (Note: the binned plan still drops this predicate per + bug #94; the dialect now sidesteps that bug entirely by + handling extras directly.) + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p_high", 100, "+"), + ("chr1", 300, 400, "p_low", 1, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g_low", 5, "+"), + ("chr1", 350, 450, "g_high", 50, "+"), + ], + ) + sql = transpile( + """ + SELECT a.start AS a_start, b.start AS b_start + FROM peaks a JOIN genes b ON a.score > b.score + WHERE a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + # Only ``p_high`` (score 100) overlaps ``g_low`` (score 5) AND beats + # its score; ``p_low`` (score 1) overlaps ``g_high`` (score 50) but + # fails the score predicate. + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100, 150)] + + def test_transpile_should_inline_extra_join_on_predicate_when_dialect_is_duckdb( + self, + ): + """Test that an extra ON predicate appears inside each per-chromosome subquery. + + Given: + A column-to-column INTERSECTS INNER join whose ON also + carries ``a.score > 20`` (a single-side predicate). + When: + The query is transpiled with ``dialect='duckdb'``. + Then: + The emitted SQL should engage the dialect path (no fallback) + and inline the extra predicate into the per-chromosome + subquery's ON, rewritten against the inner ``a``/``b`` scope. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval AND a.score > 20", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # The score predicate must show up inside the dynamic SQL literal so + # DuckDB evaluates it inside each per-chromosome IEJoin candidate set. + assert "a.score > 20" in sql + + def test_query_should_filter_by_extra_where_predicate_when_dialect_is_duckdb( + self, conn + ): + """Test that a WHERE predicate alongside an ON-INTERSECTS filters rows. + + Given: + Two overlap rows on the same chromosome and an extra + ``WHERE b.score < 30`` predicate that should exclude one + of them. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + Only the row whose ``b.score`` satisfies the predicate is + returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 300, 400, "p2", 2, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g_low", 5, "+"), + ("chr1", 350, 450, "g_high", 50, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE b.score < 30", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100,)] + + def test_query_should_combine_multiple_extra_predicates_when_dialect_is_duckdb( + self, conn + ): + """Test that two extra predicates AND together correctly under the dialect. + + Given: + Three peaks/genes with varying scores, and ``ON ... AND a.score > 20`` + plus ``WHERE b.score < 40`` — only one pair satisfies both. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + Only the pair satisfying both extras is returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 10, "+"), # score too low for ON + ("chr1", 300, 400, "p2", 25, "+"), # score passes ON + ("chr1", 500, 600, "p3", 30, "+"), # score passes ON + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 5, "+"), + ("chr1", 350, 450, "g2", 5, "+"), # b.score passes WHERE + ("chr1", 550, 650, "g3", 50, "+"), # b.score fails WHERE + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval AND a.score > 20 " + "WHERE b.score < 40", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(300,)] + + def test_transpile_should_route_to_binned_plan_when_extra_predicate_uses_or( + self, + ): + """Test that an OR-wrapped extra predicate falls back to the binned plan. + + Given: + A column-to-column INTERSECTS INNER join whose JOIN ON wraps + the INTERSECTS and a sibling predicate in an OR. + When: + The query is transpiled with ``dialect='duckdb'``. + Then: + The dialect cannot peel the INTERSECTS out of the OR tree + (only AND-conjunctions are decomposable safely) and routes + to the binned plan. + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval OR a.score > 0", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_raise_when_extra_predicate_is_unqualified( + self, + ): + """Test that an unqualified-column extra predicate raises a ValueError. + + Given: + A column-to-column INTERSECTS INNER join carrying + ``WHERE strand = '+'`` (the column reference has no table + qualifier). + When: + The query is transpiled with ``dialect='duckdb'``. + Then: + The dialect cannot safely scope the predicate to either + ``a`` or ``b`` inside the inner subquery, and the binned + plan can't handle it correctly either (issue #94), so the + dialect raises with an actionable message naming the join's + valid aliases instead of silently routing to a broken plan. + """ + # Act & assert + with pytest.raises(ValueError, match="qualified with 'a' or 'b'"): + transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE strand = '+'", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + def test_transpile_should_rewrite_residual_aliases_to_inner_a_b_when_user_uses_other_aliases( + self, + ): + """Test that residuals are rewritten when the user picks aliases other than a/b. + + Given: + A column-to-column INTERSECTS INNER join whose user aliases + are ``p`` and ``g`` (instead of the dialect's hardcoded inner + ``a`` / ``b``) and an extra ON predicate ``p.score > g.score``. + When: + The query is transpiled with ``dialect='duckdb'``. + Then: + The emitted inner SQL should reference ``a`` and ``b`` rather + than ``p`` and ``g`` in the extra predicate (otherwise the + inner subquery would fail to resolve the column refs). + """ + # Arrange & act + sql = transpile( + "SELECT p.start AS s FROM peaks p " + "JOIN genes g ON p.interval INTERSECTS g.interval AND p.score > g.score", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "a.score > b.score" in sql + assert "p.score" not in sql + assert "g.score" not in sql + + def test_transpile_should_handle_custom_chrom_col_when_dialect_is_duckdb(self, conn): + """Test that custom column names round-trip through the IEJoin path. + + Given: + Two DuckDB tables defined with ``chromosome`` / ``start_pos`` / + ``end_pos`` instead of the default column names, and a + qualified-projection query referring to those columns. + When: + The query is transpiled with the matching ``Table`` configs and + ``dialect='duckdb'`` and executed against the connection. + Then: + It should return the overlap rows correctly using the custom + column names on both sides of the join. + """ + # Arrange + conn.execute( + "CREATE TABLE peaks (chromosome VARCHAR, start_pos INTEGER, end_pos INTEGER)" + ) + conn.execute("INSERT INTO peaks VALUES ('chr1', 100, 200), ('chr1', 1000, 1100)") + conn.execute( + "CREATE TABLE genes (chromosome VARCHAR, start_pos INTEGER, end_pos INTEGER)" + ) + conn.execute("INSERT INTO genes VALUES ('chr1', 150, 250), ('chr1', 5000, 6000)") + sql = transpile( + """ + SELECT a.chromosome AS a_chrom, a.start_pos AS a_start, + b.start_pos AS b_start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table( + "peaks", + chrom_col="chromosome", + start_col="start_pos", + end_col="end_pos", + ), + Table( + "genes", + chrom_col="chromosome", + start_col="start_pos", + end_col="end_pos", + ), + ], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert rows == [("chr1", 100, 150)] + + def test_transpile_should_passthrough_literal_intersects_when_dialect_is_duckdb( + self, conn + ): + """Test that literal-range INTERSECTS bypasses the IEJoin scaffolding. + + Given: + A WHERE literal-range INTERSECTS query (not column-to-column). + When: + The query is transpiled with ``dialect=None`` and with + ``dialect='duckdb'`` and both are executed. + Then: + It should produce the same SQL and the same result rows under + both dialects (the IEJoin path is a no-op for literal ranges). + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 0, "+"), + ("chr1", 1500, 1700, "p2", 0, "+"), + ("chr1", 5000, 6000, "p3", 0, "+"), + ], + ) + query = "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'" + + # Act + sql_default = transpile(query, tables=["peaks"]) + sql_duckdb = transpile(query, tables=["peaks"], dialect="duckdb") + rows_default = sorted(conn.execute(sql_default).fetchall()) + rows_duckdb = sorted(conn.execute(sql_duckdb).fetchall()) + + # Assert + assert "SET VARIABLE" not in sql_duckdb.upper() + assert "getvariable" not in sql_duckdb + assert rows_default == rows_duckdb + assert rows_duckdb == [("chr1", 1500, 1700, "p2", 0, "+")] + + def test_transpile_should_raise_when_dialect_is_unknown(self): + """Test that an unknown dialect string raises ``ValueError``. + + Given: + A dialect name other than ``'duckdb'`` or ``None``. + When: + ``transpile`` is called. + Then: + It should raise ``ValueError`` with a message that names the + unknown dialect. + """ + # Arrange + query = "SELECT * FROM peaks" + + # Act & assert — message must name both the failure mode and the + # offending value so the user can recover without guessing. + with pytest.raises(ValueError, match=r"[Uu]nknown dialect.*'postgres'"): + transpile(query, tables=["peaks"], dialect="postgres") + + def test_transpile_should_raise_when_select_has_unqualified_wildcard(self): + """Test that bare ``SELECT *`` is rejected under the DuckDB dialect. + + Given: + A column-to-column INTERSECTS JOIN whose projection is a bare + ``*``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` whose message mentions + "qualified" projections. + """ + # Arrange + query = """ + SELECT * + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act & assert + with pytest.raises(ValueError, match="qualified"): + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + def test_transpile_should_raise_when_dialect_duckdb_and_intersects_bin_size_are_both_set( + self, + ): + """Test that ``dialect='duckdb'`` and ``intersects_bin_size`` are mutually exclusive. + + Given: + ``dialect='duckdb'`` together with a non-None + ``intersects_bin_size``. + When: + ``transpile`` is called. + Then: + It should raise ``ValueError`` whose message names + ``intersects_bin_size``. + """ + # Arrange + query = """ + SELECT a.chrom, a.start, b.start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act & assert + with pytest.raises(ValueError, match="intersects_bin_size"): + transpile( + query, + tables=["peaks", "genes"], + dialect="duckdb", + intersects_bin_size=50000, + ) + + # --- DI-001..DI-009: new SQL-structure / contract tests -------------- + + def test_transpile_should_be_round_trip_executable_when_dialect_is_duckdb( + self, peaks_genes + ): + """Test end-to-end execution of an INNER JOIN INTERSECTS under DuckDB. + + Given: + A column-to-column INTERSECTS INNER JOIN with default column + names against a multi-chromosome fixture with a known truth + table of overlapping pairs. + When: + The query is transpiled with ``dialect='duckdb'`` and executed + against an in-memory DuckDB connection. + Then: + It should return exactly the expected set of overlapping + (peak, gene) tuples. + """ + # Arrange + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(peaks_genes.execute(sql).fetchall()) + + # Assert + assert rows == sorted(_EXPECTED_OVERLAPS_PEAKS_GENES) + + def test_transpile_should_route_to_binned_plan_when_query_has_two_intersects_predicates( + self, conn + ): + """Test that two INTERSECTS predicates fall back to the binned plan. + + Given: + A three-table chained INTERSECTS query (peaks JOIN genes + JOIN genes2) with ON-INTERSECTS predicates on each join. + When: + The query is transpiled both with ``dialect='duckdb'`` and + with ``dialect=None`` and both are executed. + Then: + The DuckDB-dialect path should fall back to the binned plan + and return the same multiset of triply-overlapping rows as the + default ``dialect=None`` path. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 150, 250, "g", 0, "+")]) + conn.execute( + "CREATE TABLE genes2 (" + 'chrom VARCHAR, "start" INTEGER, "end" INTEGER, ' + "name VARCHAR, score INTEGER, strand VARCHAR)" + ) + conn.execute("INSERT INTO genes2 VALUES ('chr1', 175, 350, 'g2', 0, '+')") + query = """ + SELECT a.start AS a_s, b.start AS b_s, c.start AS c_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + JOIN genes2 c ON b.interval INTERSECTS c.interval + """ + sql_dd = transpile( + query, + tables=["peaks", "genes", "genes2"], + dialect="duckdb", + ) + sql_default = transpile(query, tables=["peaks", "genes", "genes2"]) + + # Act + rows_dd = sorted(conn.execute(sql_dd).fetchall()) + rows_default = sorted(conn.execute(sql_default).fetchall()) + + # Assert + assert rows_dd == rows_default + assert rows_dd == [(100, 150, 175)] + + def test_transpile_should_route_self_join_to_fallback_plan_when_dialect_is_duckdb( + self, conn + ): + """Test that a self-join INTERSECTS falls back and still executes. + + Given: + A query joining ``peaks`` against itself on INTERSECTS. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should fall back to the binned plan and return the + self-overlap pairs without raising. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 0, "+"), + ("chr1", 150, 300, "p2", 0, "+"), + ("chr1", 1000, 1100, "p3", 0, "+"), + ], + ) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a JOIN peaks b ON a.interval INTERSECTS b.interval + """, + tables=["peaks"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + # Each peak overlaps itself; p1[100,200) and p2[150,300) overlap each + # other; p3 only overlaps itself. + assert rows == [ + (100, 100), + (100, 150), + (150, 100), + (150, 150), + (1000, 1000), + ] + + def test_transpile_should_emit_order_by_and_limit_on_outer_select_when_dialect_is_duckdb( + self, conn + ): + """Test that ORDER BY and LIMIT survive on the dialect's outer SELECT. + + Given: + A column-to-column INTERSECTS INNER join carrying a top-level + ``ORDER BY a.start LIMIT 1``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should engage the dialect path (``SET VARIABLE + __giql_iejoin_`` is emitted), append the user's ``ORDER BY`` + / ``LIMIT`` to the outer SELECT, and return exactly the + first row by ascending ``a.start``. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 300, 400, "p2", 2, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start LIMIT 1", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # Structural check: ORDER BY must sit on the outer wrapper (after + # ``query(getvariable(...)``) and before ``LIMIT`` — pins the clause + # ordering contract of ``_build_sql``. + wrapper_idx = sql.index("query(getvariable(") + order_idx = sql.index("ORDER BY", wrapper_idx) + limit_idx = sql.index("LIMIT 1", wrapper_idx) + assert wrapper_idx < order_idx < limit_idx + assert rows == [(100,)] + + def test_transpile_should_emit_distinct_on_outer_select_when_dialect_is_duckdb( + self, conn + ): + """Test that SELECT DISTINCT survives on the dialect's outer SELECT. + + Given: + A column-to-column INTERSECTS INNER join projecting + ``DISTINCT a.chrom`` over fixture data with two overlapping + peaks sharing the same chromosome. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should engage the dialect path (``SET VARIABLE + __giql_iejoin_`` is emitted), prepend ``DISTINCT`` to the + outer SELECT, and return exactly one distinct chromosome row. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 300, 400, "p2", 2, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT DISTINCT a.chrom AS c FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "SELECT DISTINCT" in sql + assert rows == [("chr1",)] + + def test_query_should_return_rows_in_order_by_order_when_dialect_is_duckdb( + self, conn + ): + """Test that ORDER BY DESC over multiple chromosomes orders rows. + + Given: + A column-to-column INTERSECTS INNER join with two overlap + rows on different chromosomes and a top-level + ``ORDER BY a.start DESC``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The returned rows should be sorted by ``a.start`` descending. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr2", 500, 600, "p2", 2, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr2", 550, 650, "g2", 2, "-"), + ], + ) + sql = transpile( + "SELECT a.start AS s, a.chrom AS c FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start DESC", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(500, "chr2"), (100, "chr1")] + + def test_query_should_respect_offset_when_dialect_is_duckdb(self, conn): + """Test that LIMIT N OFFSET M skips the first M rows. + + Given: + Two overlap rows after sorting by ``a.start``. + When: + The query is transpiled with ``dialect='duckdb'``, a + ``LIMIT 1 OFFSET 1`` is included, and the SQL is executed. + Then: + The second-by-order row is returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 300, 400, "p2", 2, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start LIMIT 1 OFFSET 1", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(300,)] + + def test_query_should_resolve_order_by_user_alias_when_dialect_is_duckdb(self, conn): + """Test that ORDER BY can reference a SELECT-list user alias directly. + + Given: + A column-to-column INTERSECTS INNER join with + ``SELECT a.start AS s ... ORDER BY s``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The order-by reference to the bare user alias should resolve + against the outer wrapper and return rows in ascending order + by ``a.start``. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 300, 400, "p2", 2, "+"), + ("chr1", 100, 200, "p1", 1, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY s", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100,), (300,)] + + def test_query_should_auto_project_order_by_column_when_not_in_select_under_dialect( + self, conn + ): + """Test that an ORDER BY column not in the SELECT is auto-projected internally. + + Given: + Two overlap rows sortable by ``b.score`` (which is not in + the user's SELECT list). + When: + The query is transpiled with ``dialect='duckdb'`` and + ``ORDER BY b.score``, and the SQL is executed. + Then: + The dialect should silently project ``b.score`` into the + inner subquery so the outer ORDER BY can resolve it; the + user's SELECT list remains unchanged, and rows return in + ``b.score`` order. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 300, 400, "p2", 2, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g_high", 99, "+"), + ("chr1", 350, 450, "g_low", 1, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY b.score", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert: ordered by g_low (b.score=1) first then g_high (b.score=99) + assert "SET VARIABLE __giql_iejoin_" in sql + assert [r[0] for r in rows] == [300, 100] + # The user's projection is still single-column ``s``. + assert all(len(r) == 1 for r in rows) + + def test_query_should_allow_order_by_when_outer_name_is_shared_under_dialect( + self, conn + ): + """Test that ORDER BY on a shared outer name resolves unambiguously. + + Given: + A column-to-column INTERSECTS INNER join projecting both + ``a.start`` and ``b.start`` (both would otherwise surface + under the outer name ``"start"``) and ``ORDER BY a.start``. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + The rewriter resolves the reference to the inner alias for + ``a.start`` rather than to a colliding outer name, so the + query succeeds and orders by ``a.start``. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 300, 400, "p2", 2, "+"), + ("chr1", 100, 200, "p1", 1, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT a.start, b.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100, 150), (300, 350)] + + def test_query_should_group_overlap_pairs_by_chrom_when_dialect_is_duckdb( + self, conn + ): + """Test that an outer GROUP BY with COUNT(*) under the dialect groups rows. + + Given: + A column-to-column INTERSECTS INNER join with a top-level + ``GROUP BY a.chrom`` and a ``COUNT(*)`` aggregate over two + chromosomes each having a single overlap. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The dialect path should engage and return one row per + chromosome with the matching count. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr2", 100, 200, "p2", 2, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr2", 150, 250, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, COUNT(*) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "GROUP BY" in sql + assert rows == [("chr1", 1), ("chr2", 1)] + + def test_query_should_filter_groups_via_having_when_dialect_is_duckdb(self, conn): + """Test that an outer HAVING filter survives under the dialect. + + Given: + A column-to-column INTERSECTS INNER join with a + ``GROUP BY ... HAVING COUNT(*) > 1`` over data where exactly + one chromosome has more than one overlap. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The dialect path should engage and return only the + chromosome whose overlap count exceeds the threshold. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 110, 210, "p2", 2, "+"), + ("chr2", 100, 200, "p3", 3, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr2", 150, 250, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, COUNT(*) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom HAVING COUNT(*) > 1", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "HAVING" in sql + assert rows == [("chr1", 2)] + + def test_query_should_compute_sum_aggregate_with_group_by_when_dialect_is_duckdb( + self, conn + ): + """Test that SUM(a.score) over GROUP BY a.chrom works under the dialect. + + Given: + A column-to-column INTERSECTS INNER join over two chromosomes + with multiple overlaps per side, plus a ``GROUP BY a.chrom`` + and ``SUM(a.score)`` aggregate. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The dialect path should engage and return one row per + chromosome with the per-chromosome sum. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 10, "+"), + ("chr1", 300, 400, "p2", 5, "+"), + ("chr2", 100, 200, "p3", 7, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr1", 350, 450, "g2", 0, "+"), + ("chr2", 150, 250, "g3", 0, "-"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, SUM(a.score) AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", 15), ("chr2", 7)] + + def test_query_should_handle_count_distinct_aggregate_when_dialect_is_duckdb( + self, conn + ): + """Test that COUNT(DISTINCT a.x) preserves DISTINCT through the rewrite. + + Given: + Overlap data where the same ``a.name`` appears twice on the + same chromosome and a ``COUNT(DISTINCT a.name)`` aggregate. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The dialect path should engage and the count should reflect + the number of distinct names (not the total row count). + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p_dup", 1, "+"), + ("chr1", 110, 210, "p_dup", 2, "+"), + ("chr1", 300, 400, "p_other", 3, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr1", 350, 450, "g2", 0, "+"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, COUNT(DISTINCT a.name) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + # The two p_dup peaks both overlap g1 — under DISTINCT they + # collapse to 1; p_other overlaps g2; total distinct = 2. + assert "SET VARIABLE __giql_iejoin_" in sql + assert "DISTINCT" in sql.upper() + assert rows == [("chr1", 2)] + + def test_query_should_match_binned_plan_avg_aggregate_when_dialect_is_duckdb( + self, conn + ): + """Test that AVG aggregate matches the binned plan executing-side. + + Given: + Two chromosomes with multiple overlap pairs and varied + ``a.score`` values that produce a non-integer per-chrom + average. + When: + A ``SELECT a.chrom, AVG(a.score) ... GROUP BY a.chrom`` is + transpiled under both ``dialect=None`` (binned plan) and + ``dialect='duckdb'`` and executed. + Then: + Both plans return the same per-chrom averages (compared via + ``pytest.approx`` since AVG yields floats). This is the + execution-side AVG-equivalence check that the aggregate PBT + intentionally skips (PBT excludes AVG to keep its equality + assertion strict). + """ + # Arrange — scores chosen so chr1's mean is 31.0 (integer) and + # chr2's mean is 12.5 (genuinely non-integer), forcing + # ``pytest.approx`` to actually do float comparison. + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 10, "+"), + ("chr1", 110, 210, "p2", 32, "+"), + ("chr1", 120, 220, "p3", 51, "+"), + ("chr2", 100, 200, "p4", 7, "-"), + ("chr2", 110, 210, "p5", 18, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr2", 150, 250, "g2", 0, "-"), + ], + ) + query = ( + "SELECT a.chrom AS c, AVG(a.score) AS v FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom ORDER BY a.chrom" + ) + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = conn.execute(sql_default).fetchall() + rows_duckdb = conn.execute(sql_duckdb).fetchall() + + # Assert — value-equivalence with float tolerance. + assert len(rows_default) == len(rows_duckdb) + for (c_d, v_d), (c_q, v_q) in zip(rows_default, rows_duckdb): + assert c_d == c_q + assert v_d == pytest.approx(v_q) + + def test_query_should_combine_count_star_and_sum_when_dialect_is_duckdb(self, conn): + """Test that multiple aggregates coexist correctly under the dialect. + + Given: + Two chromosomes with multiple overlaps and a SELECT list that + mixes ``COUNT(*)`` and ``SUM(b.score)``. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The dialect should engage, project the underlying column for + ``SUM(b.score)`` into the inner subquery, and emit both + aggregates against the wrapper relation. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 110, 210, "p2", 2, "+"), + ("chr2", 100, 200, "p3", 3, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 10, "+"), + ("chr2", 150, 250, "g2", 20, "-"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, COUNT(*) AS n, SUM(b.score) AS bs " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # chr1: two overlaps, each with g1 (score 10) -> sum 20; chr2: 1 with g2 (20) + assert rows == [("chr1", 2, 20), ("chr2", 1, 20)] + + def test_transpile_should_raise_when_aggregate_argument_is_unqualified(self): + """Test that an aggregate with an unqualified-column argument raises. + + Given: + A SELECT-list aggregate ``SUM(score)`` whose argument is not + table-qualified. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + A ``ValueError`` should be raised that names the offending + projection and asks for a qualified reference. + """ + # Act & assert + with pytest.raises(ValueError, match=r"qualified"): + transpile( + "SELECT a.chrom AS c, SUM(score) AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + def test_transpile_should_route_to_binned_plan_when_outer_join_intersects_lives_in_where( + self, + ): + """Test that LEFT JOIN ON TRUE WHERE INTERSECTS falls back. + + Given: + A ``LEFT JOIN ... ON TRUE WHERE a.interval INTERSECTS + b.interval`` query — the INTERSECTS lives in WHERE while + the join keeps its LEFT side modifier. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect must NOT engage (rewriting as an inner IEJoin + would silently drop LEFT JOIN's unmatched-row guarantee). + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "LEFT JOIN genes b ON TRUE " + "WHERE a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_route_to_binned_plan_when_full_outer_join_intersects_lives_in_where( + self, + ): + """Test that FULL OUTER JOIN ... WHERE INTERSECTS falls back. + + Given: + A ``FULL OUTER JOIN ... ON TRUE WHERE a.interval INTERSECTS + b.interval`` — same shape as LEFT, but FULL. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect must not engage; the binned plan handles the + outer-join semantics. + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "FULL OUTER JOIN genes b ON TRUE " + "WHERE a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_route_to_binned_plan_when_extra_predicate_contains_window_function( + self, + ): + """Test that window functions inside extras force a fallback. + + Given: + An INTERSECTS join carrying ``WHERE a.score > (ROW_NUMBER() + OVER (PARTITION BY a.chrom))`` — a window function in an + extra predicate. DuckDB forbids window functions in JOIN ON + clauses, so the dialect cannot inline this residual. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect must reject the residual and route to the + binned plan (rather than emit SQL that DuckDB rejects at + runtime). + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE a.score > (ROW_NUMBER() OVER (PARTITION BY a.chrom))", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_raise_with_window_specific_message_when_window_aggregate_in_select( + self, + ): + """Test that a window-aggregate projection raises with a targeted message. + + Given: + A SELECT list containing ``SUM(a.score) OVER (PARTITION BY + a.chrom)`` — a window aggregate, which the dispatcher does + not classify as ``exp.AggFunc``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + A ``ValueError`` is raised whose message names "window + aggregates" (rather than the generic "qualified + projections" message). + """ + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + "SELECT a.chrom AS c, " + "SUM(a.score) OVER (PARTITION BY a.chrom) AS s " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + assert "window" in str(excinfo.value).lower() + + def test_transpile_should_raise_with_window_message_when_paren_wrapped_window_aggregate( + self, + ): + """Test that ``(SUM(a.score) OVER (...))`` gets the targeted window message. + + Given: + A paren-wrapped window aggregate in the SELECT list. The + Paren wrapper hides the inner ``exp.Window`` from the + non-peeled type check that the diagnostic branches used to + rely on. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dispatcher peels the Paren and routes to the window + diagnostic branch, raising the dedicated window-aggregate + message rather than the generic + arithmetic-over-aggregate one. + """ + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + "SELECT a.chrom AS c, " + "(SUM(a.score) OVER (PARTITION BY a.chrom)) AS s " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + assert "window" in str(excinfo.value).lower() + + def test_transpile_should_raise_with_aggregate_in_expression_message_when_count_in_arithmetic( + self, + ): + """Test that arithmetic over an aggregate raises with a targeted message. + + Given: + A SELECT list containing ``COUNT(*) * 2`` — arithmetic over + an aggregate, which the dispatcher does not classify as + ``exp.AggFunc``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + A ``ValueError`` is raised whose message names aggregates + inside expressions (rather than the generic "qualified + projections" message). + """ + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + "SELECT a.chrom AS c, COUNT(*) * 2 AS doubled " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + message = str(excinfo.value).lower() + assert "aggregate" in message and ( + "expression" in message or "arithmetic" in message + ) + + def test_transpile_should_route_to_binned_plan_when_modifier_contains_subquery( + self, + ): + """Test that a subquery inside ORDER BY routes to the binned plan. + + Given: + A query whose ``ORDER BY`` clause embeds a scalar subquery + (``(SELECT MAX(score) FROM peaks)``). The modifier rewriter + is not scope-aware, so the dialect should fall back to the + binned plan rather than rewrite column refs inside the + nested scope. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect must not engage; the binned plan handles the + subquery natively. + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY (SELECT MAX(score) FROM peaks)", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_route_to_binned_plan_when_modifier_contains_exists( + self, + ): + """Test that EXISTS inside ORDER BY routes to the binned plan. + + Given: + A query whose ``ORDER BY`` clause embeds an ``EXISTS + (SELECT ...)`` clause. sqlglot parses EXISTS as + ``Exists(Select(...))`` without an ``exp.Subquery`` + wrapper, so the gate must also check for ``exp.Select`` — + otherwise EXISTS slips past and the non-scope-aware + modifier rewriter corrupts the EXISTS scope. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect must not engage. + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY EXISTS (SELECT 1 FROM peaks p WHERE p.score > 0)", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_route_to_binned_plan_when_having_contains_subquery( + self, + ): + """Test that a subquery inside HAVING routes to the binned plan. + + Given: + A query whose HAVING compares against a scalar subquery + (``HAVING SUM(a.score) > (SELECT AVG(score) FROM peaks)``). + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect must not engage. + """ + # Arrange & act + sql = transpile( + "SELECT a.chrom AS c, SUM(a.score) AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom " + "HAVING SUM(a.score) > (SELECT AVG(score) FROM peaks)", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_accept_paren_wrapped_aggregate_in_select( + self, + ): + """Test that ``(SUM(a.score)) AS s`` is accepted, not rejected. + + Given: + A SELECT-list projection that wraps an aggregate in an + otherwise-redundant ``exp.Paren`` (``(SUM(a.score)) AS s``). + Previously this hit the arithmetic-over-aggregate raise + with misleading guidance, because ``exp.Paren`` is not an + ``exp.AggFunc``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect should peel the paren and route the projection + through the aggregate branch, emitting valid SQL with the + inner-alias rewrite. + """ + # Arrange & act + sql = transpile( + "SELECT a.chrom AS c, (SUM(a.score)) AS s " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # The aggregate's argument should be rewritten to an inner alias + # rather than left as ``a.score`` on the outer SELECT. + outer_select = sql.split(";\n", 1)[-1] + assert "a.score" not in outer_select + assert re.search(r"SUM\(__giql_p\d+\)", outer_select) is not None + + def test_transpile_should_raise_with_subquery_specific_message_when_scalar_subquery_in_select( + self, + ): + """Test that a scalar subquery in SELECT gets a targeted error. + + Given: + A SELECT list containing a scalar subquery that itself + contains an aggregate (``(SELECT SUM(score) FROM + other_table)``). Previously this hit the + arithmetic-over-aggregate message with misleading guidance. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + A ``ValueError`` whose message names the scalar-subquery + shape (rather than steering the user toward "do the + arithmetic in a wrapping query"). + """ + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + "SELECT a.chrom AS c, " + "(SELECT SUM(score) FROM peaks) AS s " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + assert "subquer" in str(excinfo.value).lower() + + def test_transpile_should_route_to_binned_plan_when_query_has_distinct_on(self): + """Test that DISTINCT ON (...) falls back to the binned plan. + + Given: + A column-to-column INTERSECTS INNER join with a + ``SELECT DISTINCT ON (a.chrom) ...`` projection. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect should not engage; the SQL should not contain + the ``SET VARIABLE __giql_iejoin_`` marker. + """ + # Arrange & act + sql = transpile( + "SELECT DISTINCT ON (a.chrom) a.chrom, a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_route_to_binned_plan_and_execute_correctly_when_query_has_top_level_with_clause( + self, conn + ): + """Test that a top-level WITH clause falls back to the binned plan. + + Given: + A query whose INTERSECTS join is wrapped in a top-level + ``WITH big AS (SELECT * FROM genes)`` whose CTE is then used + as one of the joined operands. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should fall back to the binned plan, preserve the CTE + definition, and execute against the materialized CTE. The + dialect path would otherwise emit a `FROM big` referencing + a missing table. + """ + # Arrange + _make_table( + conn, + "peaks", + [("chr1", 100, 200, "p1", 1, "+")], + ) + _make_table( + conn, + "genes", + [("chr1", 150, 250, "g1", 1, "+")], + ) + sql = transpile( + "WITH big AS (SELECT * FROM genes) " + "SELECT a.start AS s FROM peaks a " + "JOIN big b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + assert rows == [(100,)] + + def test_transpile_should_route_to_binned_plan_when_query_has_three_table_cross_join( + self, conn + ): + """Test that a 3-table implicit cross-join falls back to the binned plan. + + Given: + A 3-table comma-style FROM clause where the INTERSECTS + predicate only connects two of the three tables. The third + table contributes a real cross-product factor to the result. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should fall back to the binned plan and return a row + count consistent with the full 3-way join — the dialect + path would silently drop the third table. + """ + # Arrange + _make_table( + conn, + "peaks", + [("chr1", 100, 200, "p1", 1, "+")], + ) + _make_table( + conn, + "genes", + [("chr1", 150, 250, "g1", 1, "+")], + ) + _make_table( + conn, + "extra", + [ + ("chr1", 0, 10, "e1", 0, "+"), + ("chr1", 0, 10, "e2", 0, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a, genes b, extra c " + "WHERE a.interval INTERSECTS b.interval", + tables=["peaks", "genes", "extra"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + # One (peaks, genes) overlap pair cross-joined with 2 extra rows + # = 2 result rows. The dialect path would emit only 1 (extra dropped). + assert len(rows) == 2 + + def test_transpile_should_route_to_binned_plan_when_join_target_is_a_giql_operator( + self, conn + ): + """Test that a GIQL table-function in JOIN position falls back. + + Given: + A query whose INTERSECTS join uses ``DISJOIN(genes)`` as the + right-hand operand. ``DISJOIN(...)`` parses as an + ``exp.Table`` with an empty ``name``; the pre-fix dialect + path would interpolate that empty name into broken SQL. + When: + The query is transpiled with ``dialect='duckdb'``. + Then: + The dialect path should return ``None`` (fall back) and the + emitted SQL should contain no ``SET VARIABLE`` block and no + empty ``FROM `` interpolation. (Execution is the binned + path's domain and is covered by DISJOIN's own tests.) + """ + # Arrange & Act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN DISJOIN(genes) b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + # No empty-name table interpolation leaked into the output. + assert "FROM " not in sql + assert "FROM (SELECT * FROM " not in sql + + def test_transpile_should_double_quote_table_name_and_execute_correctly_when_table_name_is_a_reserved_word( + self, conn + ): + """Test that the dialect double-quotes a reserved-word table name. + + Given: + A column-to-column INTERSECTS INNER join whose left table is + named ``select`` — a SQL reserved word that must be quoted + wherever the dialect interpolates it. + When: + The query is transpiled with ``dialect='duckdb'`` and the + generated SQL is executed against a real DuckDB table named + ``select``. + Then: + The dialect should engage (this *is* the supported shape), + the emitted SQL should reference ``"select"`` (quoted) and + never bare ``FROM select``, and the query should execute to + the expected overlap row. + """ + # Arrange + conn.execute( + 'CREATE TABLE "select" ' + '(chrom VARCHAR, "start" BIGINT, "end" BIGINT, name VARCHAR, ' + "score INTEGER, strand VARCHAR)" + ) + conn.execute( + 'INSERT INTO "select" VALUES (?, ?, ?, ?, ?, ?)', + ("chr1", 100, 200, "p", 1, "+"), + ) + conn.execute( + "CREATE TABLE genes " + '(chrom VARCHAR, "start" BIGINT, "end" BIGINT, name VARCHAR, ' + "score INTEGER, strand VARCHAR)" + ) + conn.execute( + "INSERT INTO genes VALUES (?, ?, ?, ?, ?, ?)", + ("chr1", 150, 250, "g", 1, "+"), + ) + sql = transpile( + 'SELECT a.start AS s FROM "select" a ' + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=[Table("select"), "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert '"select"' in sql + # Bare unquoted identifier in a FROM position would have broken DuckDB. + assert "FROM select " not in sql + assert "FROM select," not in sql + assert rows == [(100,)] + + def test_transpile_should_emit_iejoin_block_and_return_overlap_when_implicit_cross_join_has_swapped_alias_order( + self, conn + ): + """Test that an implicit cross-join with swapped FROM order works. + + Given: + An implicit cross-join where the FROM clause lists ``genes`` + first and ``peaks`` second, while the INTERSECTS predicate + references ``a.interval`` (peaks) before ``b.interval`` + (genes). + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + It should emit the dialect's ``SET VARIABLE`` IEJoin block + and return the correct overlap pair (the IEJoin path must + not depend on FROM order matching INTERSECTS argument order). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 150, 250, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM genes b, peaks a + WHERE a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE" in sql.upper() + assert rows == [(100, 150)] + + def test_transpile_should_raise_when_select_list_has_only_unqualified_columns( + self, + ): + """Test that unqualified column projections are rejected under DuckDB. + + Given: + A query whose SELECT list contains bare column names (no + table qualifier). + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` whose message mentions + "qualified". + """ + # Arrange + query = """ + SELECT chrom, start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act & assert — message should name the offending column and the + # ``a.col``/``b.col`` form so the user can fix the query directly. + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + message = str(excinfo.value) + assert "qualified" in message + assert "chrom" in message + assert "a.col" in message or "b.col" in message + + def test_transpile_should_raise_when_projection_references_unknown_table_alias( + self, + ): + """Test that a projection referring to an out-of-scope alias is rejected. + + Given: + A SELECT list that qualifies a column with a table alias + (``c``) not present in the FROM/JOIN clauses. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` whose message names the + unknown qualifier. + """ + # Arrange + query = """ + SELECT c.col + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act & assert + with pytest.raises(ValueError, match="[Uu]nknown.*qualifier|'c'"): + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + def test_transpile_should_raise_when_projection_uses_expression_form(self): + """Test that an arithmetic projection expression is rejected. + + Given: + A SELECT list containing an expression + (``a.start + 1``) rather than a bare qualified column. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` whose message mentions + "qualified". + """ + # Arrange + query = """ + SELECT a.start + 1 + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act & assert — message should echo the offending expression so + # the user can spot which projection to rewrite. + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + message = str(excinfo.value) + assert "qualified" in message + assert "a.start + 1" in message or "a.start" in message + + def test_transpile_should_raise_when_star_qualifier_references_unknown_table( + self, + ): + """Test that ``c.*`` against an out-of-scope alias is rejected. + + Given: + A SELECT list with a qualified-star projection ``c.*`` whose + qualifier ``c`` is not present in the FROM/JOIN clauses. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` whose message names the + unknown qualifier. + """ + # Arrange + query = """ + SELECT c.* + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act & assert + with pytest.raises(ValueError, match="[Uu]nknown.*qualifier|'c'"): + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + def test_transpile_should_use_unique_variable_names_across_interleaved_calls( + self, conn + ): + """Test that two interleaved transpile outputs do not collide. + + Given: + Two ``transpile(..., dialect='duckdb')`` outputs whose + ``SET VARIABLE`` and ``SELECT`` statements are interleaved + (both ``SET``, then both ``SELECT``) in a single DuckDB + session, with the two queries projecting different columns + so that a fixed-name collision would surface one query's + result in the other's slot. + When: + The interleaved statements are executed. + Then: + Each ``SELECT`` should return the result of its own query — + proving the dialect emits a per-call unique variable name. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 150, 250, "g", 0, "+")]) + sql_a = transpile( + "SELECT a.chrom AS v FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + sql_b = transpile( + "SELECT a.start AS v FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + # The dialect emits exactly one `;\n` between the SET and the + # SELECT, so split each output on that to interleave the four + # statements deliberately. Under a single fixed variable name + # the second `SET` would overwrite the first before either + # `SELECT` ran, surfacing query B's row in query A's slot. + set_a, sep_a, select_a = sql_a.partition(";\n") + set_b, sep_b, select_b = sql_b.partition(";\n") + assert sep_a and sep_b + assert set_a.startswith("SET VARIABLE") + assert set_b.startswith("SET VARIABLE") + assert select_a.startswith("SELECT") + assert select_b.startswith("SELECT") + + # Act + conn.execute(set_a) + conn.execute(set_b) + row_a = conn.execute(select_a).fetchall() + row_b = conn.execute(select_b).fetchall() + + # Assert + assert row_a == [("chr1",)] + assert row_b == [(100,)] + + def test_transpile_should_preserve_order_by_modifiers_when_dialect_is_duckdb( + self, + ): + """Test that DESC and NULLS FIRST modifiers survive the rewrite. + + Given: + A column-to-column INTERSECTS join with + ``ORDER BY a.start DESC NULLS FIRST``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The emitted outer SELECT should preserve both ``DESC`` and + ``NULLS FIRST`` in the rewritten ORDER BY clause. (sqlglot + strips its own defaults like ``ASC NULLS FIRST``, so the + test uses the non-default-for-DESC variant to assert that + both modifiers round-trip through the rewriter.) + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start DESC NULLS FIRST", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + outer_select = sql.split(";\n", 1)[-1] + order_idx = outer_select.index("ORDER BY") + assert "DESC" in outer_select[order_idx:] + assert "NULLS FIRST" in outer_select[order_idx:] + + def test_transpile_should_preserve_multiple_order_by_expressions_when_dialect_is_duckdb( + self, + ): + """Test that multi-column ORDER BY survives, with per-term modifiers. + + Given: + A column-to-column INTERSECTS join with + ``ORDER BY a.start, b.end DESC``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The outer ORDER BY clause should carry two comma-separated + terms, ``DESC`` should attach to the second term only, and + neither the user's ``a.``/``b.`` qualifiers should remain on + the outer ORDER BY (they get rewritten to inner aliases). + """ + # Arrange & act + sql = transpile( + "SELECT a.start AS s, b.end AS e FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start, b.end DESC", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + wrapper_idx = sql.index("query(getvariable(") + order_clause = sql[sql.index("ORDER BY", wrapper_idx) :] + # Pin the DESC attachment positionally so a regression that + # accidentally moved DESC to the first term (or attached it to + # both) would fail. Two inner-alias placeholders, comma-separated, + # with DESC on the second. + assert ( + re.search( + r"ORDER BY\s+__giql_p\d+(?:\s+NULLS\s+(?:FIRST|LAST))?\s*," + r"\s*__giql_p\d+\s+DESC", + order_clause, + ) + is not None + ) + # The rewriter binds user-qualified refs to inner aliases on the + # outer SELECT, so ``a.start`` / ``b.end`` should not appear in + # the outer ORDER BY. + assert "a.start" not in order_clause + assert "b.end" not in order_clause + + def test_transpile_should_inline_cross_side_on_predicate_when_dialect_is_duckdb( + self, + ): + """Test that a cross-side ON predicate is inlined into each subquery. + + Given: + A join ``ON a.interval INTERSECTS b.interval AND a.score > b.score`` + referencing both sides of the inner subquery. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The cross-side predicate ``a.score > b.score`` should appear + in the dynamic SQL literal (inside the per-chromosome + subquery template), proving both ``a`` and ``b`` are in + scope inside the inlined extra predicate. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "AND a.score > b.score", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "a.score > b.score" in sql + + def test_transpile_should_inline_between_predicate_when_dialect_is_duckdb( + self, + ): + """Test that a ``BETWEEN`` extra predicate is inlined. + + Given: + A join carrying an extra ``BETWEEN`` predicate on a + qualified column. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The ``BETWEEN 10 AND 50`` substring should appear in the + dynamic SQL literal. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "AND a.score BETWEEN 10 AND 50", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert — the predicate must land inside the per-chromosome + # template (SET VARIABLE half), not on the outer wrapper SELECT. + assert "SET VARIABLE __giql_iejoin_" in sql + set_part, outer_part = sql.split(";\n", 1) + assert "BETWEEN 10 AND 50" in set_part + assert "BETWEEN" not in outer_part + + def test_transpile_should_inline_is_null_predicate_when_dialect_is_duckdb( + self, + ): + """Test that an ``IS NULL`` extra predicate is inlined. + + Given: + A join carrying ``AND a.strand IS NULL`` as an extra + predicate. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The ``IS NULL`` substring should appear in the dynamic SQL + literal. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "AND a.strand IS NULL", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert — the predicate must land inside the per-chromosome + # template (SET VARIABLE half), not on the outer wrapper SELECT. + assert "SET VARIABLE __giql_iejoin_" in sql + set_part, outer_part = sql.split(";\n", 1) + assert "IS NULL" in set_part + assert "IS NULL" not in outer_part + + def test_transpile_should_route_to_binned_plan_when_paren_wraps_intersects( + self, + ): + """Test that a Paren-wrapped INTERSECTS routes to the binned plan. + + Given: + A join ``ON (a.interval INTERSECTS b.interval) AND a.score > 0`` + where the INTERSECTS is wrapped in explicit parentheses. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect only decomposes AND-conjunctions when stripping + the INTERSECTS out of a join clause; a paren wrapper leaves + the INTERSECTS embedded in the residual and the dialect + must fall back to the binned plan. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "JOIN genes b ON (a.interval INTERSECTS b.interval) " + "AND a.score > 0", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_route_to_binned_plan_when_residual_contains_subquery( + self, + ): + """Test that a subquery in the residual predicate forces a fallback. + + Given: + A join INTERSECTS plus + ``WHERE a.score > (SELECT MAX(score) FROM peaks)``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect's residual-safety check rejects predicates + containing a subquery, so the query falls back to the + binned plan. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE a.score > (SELECT MAX(score) FROM peaks)", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_rewrite_group_by_multiple_columns_to_inner_aliases_when_dialect_is_duckdb( + self, + ): + """Test that multi-column GROUP BY rewrites every key to an inner alias. + + Given: + A query ``SELECT a.chrom, a.score, COUNT(*) ... GROUP BY a.chrom, a.score``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The outer GROUP BY clause should reference two distinct + ``__giql_p`` placeholders (one per key) and the user's + ``a.chrom``/``a.score`` should not appear in the outer + GROUP BY. + """ + + # Arrange & act + sql = transpile( + "SELECT a.chrom AS c, a.score AS s, COUNT(*) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom, a.score", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + wrapper_idx = sql.index("query(getvariable(") + outer = sql[wrapper_idx:] + group_clause = outer[outer.index("GROUP BY") :] + # Two distinct inner-alias placeholders should appear in GROUP BY. + placeholders = set( + re.findall(r"__giql_p\d+", group_clause.split("HAVING")[0].split("ORDER")[0]) + ) + assert len(placeholders) >= 2 + assert "a.chrom" not in group_clause.split("ORDER")[0].split("HAVING")[0] + assert "a.score" not in group_clause.split("ORDER")[0].split("HAVING")[0] + + def test_transpile_should_emit_sum_min_max_avg_aggregates_with_inner_alias_when_dialect_is_duckdb( + self, + ): + """Test that four aggregates over the same column reuse one inner alias. + + Given: + A query ``SELECT SUM(a.score), MIN(a.score), MAX(a.score), + AVG(a.score) ... GROUP BY a.chrom``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The inner SELECT should project ``a."score"`` exactly once, + and all four outer aggregates should reference the same + ``__giql_p`` placeholder. + """ + + # Arrange & act + sql = transpile( + "SELECT a.chrom AS c, " + "SUM(a.score) AS s, MIN(a.score) AS mn, " + "MAX(a.score) AS mx, AVG(a.score) AS av FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # Behavior-level dedup check: ``a."score"`` should be allocated to + # exactly one inner alias across the whole SQL string, regardless + # of which ``__giql_p`` index that alias happens to get + # (allocation order depends on sqlglot's traversal of the SELECT + # list and is not part of the dialect's public contract). + score_inner_allocs = re.findall(r'a\."score"\s+AS\s+(__giql_p\d+)', sql) + assert len(set(score_inner_allocs)) == 1 + outer_select = sql.split(";\n", 1)[-1] + # Each of the four aggregates appears once with an inner-alias arg, + # and all four reference the same alias (dedup propagated to the + # outer SELECT). + agg_refs = re.findall(r"(SUM|MIN|MAX|AVG)\((__giql_p\d+)\)", outer_select) + assert len(agg_refs) == 4 + assert len({alias for _, alias in agg_refs}) == 1 + # User aliases preserved on the outer SELECT. + for label in ('"s"', '"mn"', '"mx"', '"av"'): + assert label in outer_select + + def test_transpile_should_auto_project_group_by_column_into_inner_select_when_not_in_outer_select_when_dialect_is_duckdb( + self, + ): + """Test that a GROUP BY column absent from SELECT is auto-projected. + + Given: + A query ``SELECT COUNT(*) AS n FROM peaks a JOIN genes b ON + ... GROUP BY a.chrom`` where ``a.chrom`` only appears in + GROUP BY. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The inner subquery should project ``a."chrom"`` (so the + wrapper relation has it available); the outer SELECT list + should be ``COUNT(*)`` only; and GROUP BY should reference + the inner alias rather than ``a.chrom``. + """ + # Arrange & act + sql = transpile( + "SELECT COUNT(*) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert 'a."chrom"' in sql # inner projection auto-added + outer_select = sql.split(";\n", 1)[-1] + select_clause = outer_select[: outer_select.index("FROM query(")] + # Only COUNT(*) — no qualified column projection. + assert "COUNT(*)" in select_clause + assert "a.chrom" not in select_clause + # GROUP BY references inner alias. + group_clause = outer_select[outer_select.index("GROUP BY") :] + assert "a.chrom" not in group_clause + assert "__giql_p" in group_clause + + def test_transpile_should_emit_aggregate_of_expression_with_rewritten_inner_aliases_when_dialect_is_duckdb( + self, + ): + """Test that ``SUM(a.start + a.end)`` rewrites both inner columns. + + Given: + A query whose aggregate argument is an expression over two + qualified columns: ``SUM(a.start + a.end)``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The inner subquery should project both ``a."start"`` and + ``a."end"`` under distinct inner aliases, the outer + aggregate should reference both inner aliases inside the + arithmetic expression, and the original ``a.start`` / + ``a.end`` should not survive in the outer SELECT. + """ + + # Arrange & act + sql = transpile( + "SELECT a.chrom AS c, SUM(a.start + a.end) AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # Both genomic columns projected into the inner subquery. + assert 'a."start"' in sql + assert 'a."end"' in sql + outer_select = sql.split(";\n", 1)[-1] + # Outer SUM references two inner aliases under arithmetic. + sum_match = re.search(r"SUM\(__giql_p\d+\s*\+\s*__giql_p\d+\)", outer_select) + assert sum_match is not None + # The two operand aliases must be distinct. + operand_aliases = set(re.findall(r"__giql_p\d+", sum_match.group(0))) + assert len(operand_aliases) == 2 + # No raw a.start / a.end in the outer SELECT. + assert "a.start" not in outer_select + assert "a.end" not in outer_select + + def test_transpile_should_route_right_outer_join_intersects_to_binned_plan_when_dialect_is_duckdb( + self, + ): + """Test that ``RIGHT JOIN ... INTERSECTS`` falls back to the binned plan. + + Given: + A query with a ``RIGHT JOIN`` whose ON contains a + column-to-column INTERSECTS. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + ``_has_outer_join_intersects`` should detect the + outer-join side and route to the binned plan. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "RIGHT JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_route_to_binned_plan_when_join_target_is_subquery( + self, + ): + """Test that a subquery JOIN target routes to the binned plan. + + Given: + A query whose JOIN target is a parenthesised subquery, + ``JOIN (SELECT * FROM genes) b ON a.interval INTERSECTS + b.interval``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The dialect's "must be a base table" gate should fire and + route to the binned plan. + """ + # Arrange & act + sql = transpile( + "SELECT a.start FROM peaks a " + "JOIN (SELECT * FROM genes) b " + "ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_raise_when_select_projects_a_literal(self): + """Test that a literal-only SELECT-list entry raises ``ValueError``. + + Given: + A query whose SELECT list contains a bare integer literal + (``SELECT 100 FROM peaks a JOIN ...``). + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + A ``ValueError`` matching ``qualified`` should be raised + and the offending literal text should appear in the message. + """ + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + "SELECT 100 FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + message = str(excinfo.value) + assert "qualified" in message + assert "100" in message + + def test_transpile_should_raise_when_star_projection_carries_a_user_alias( + self, + ): + """Test that ``SELECT a.* AS x`` is rejected rather than silently stripped. + + Given: + A query with a star projection that also has a user alias + (``SELECT a.* AS x``). + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + A ``ValueError`` should be raised mentioning the + unsupported alias and pointing at concrete alternatives + (drop the alias, or list columns explicitly). + """ + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + "SELECT a.* AS x FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + message = str(excinfo.value).lower() + assert "star" in message or "alias" in message + + def test_transpile_should_surface_value_error_for_unqualified_projection_under_dialect( + self, + ): + """Test that the dialect surfaces a plain ``ValueError`` on bare ``*``. + + Given: + A query that triggers the dialect's unqualified-projection + rejection (bare ``SELECT *``). + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The exception surfaced to the caller should be a + ``ValueError`` and the docs-promised behavior holds: the + ``"qualified"`` keyword appears in the message, and the + class is reachable via ``isinstance(..., ValueError)`` (no + internal subclass leaks across that contract). + """ + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + "SELECT * FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + assert isinstance(excinfo.value, ValueError) + assert "qualified" in str(excinfo.value) + + def test_transpile_should_fall_back_to_binned_plan_when_join_is_natural(self): + """Test that NATURAL JOIN falls back to the binned plan. + + Given: + A query joining two tables with ``NATURAL JOIN`` and an + INTERSECTS predicate in WHERE. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should emit SQL with no ``SET VARIABLE`` scaffolding + (the dialect cannot enumerate NATURAL's implicit shared + columns at transpile time, so it falls back). + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "NATURAL JOIN genes b " + "WHERE a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_engage_dialect_when_using_join_targets_chrom_only(self): + """Test that ``USING(chrom)`` engages the dialect path. + + Given: + A query joining two tables with ``USING(chrom)`` and an + INTERSECTS predicate in WHERE — the per-chromosome partition + IS the equi-join that USING(chrom) requests. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should emit a ``SET VARIABLE __giql_iejoin_`` block. + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a JOIN genes b USING (chrom) " + "WHERE a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + + def test_transpile_should_fall_back_when_using_join_targets_non_chrom_column(self): + """Test that ``USING()`` falls back to the binned plan. + + Given: + A query with ``USING(strand)`` plus INTERSECTS — the partition + cannot satisfy a strand equi-join. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should fall back to the binned plan (no + ``SET VARIABLE __giql_iejoin_`` block). + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a JOIN genes b USING (strand) " + "WHERE a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_fall_back_when_using_join_targets_multiple_columns(self): + """Test that multi-column ``USING(chrom, strand)`` falls back. + + Given: + A query with ``USING(chrom, strand)`` plus INTERSECTS. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should fall back to the binned plan (multi-column USING + inline support is a documented follow-up). + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "JOIN genes b USING (chrom, strand) " + "WHERE a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_engage_dialect_when_join_is_implicit_cross_join(self): + """Test that CROSS JOIN + WHERE INTERSECTS continues to engage the dialect. + + Given: + A query joining two tables with ``CROSS JOIN`` and an + INTERSECTS predicate in WHERE — equivalent to the implicit + cross-join shape the dialect was designed for. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should emit a ``SET VARIABLE __giql_iejoin_`` block + (regression guard so the new gate doesn't reject the + legitimate cross-join shape). + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a CROSS JOIN genes b " + "WHERE a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + + def test_transpile_should_fall_back_when_aggregate_argument_contains_subquery( + self, + ): + """Test that a subquery inside an aggregate argument falls back. + + Given: + A query whose SELECT list contains ``SUM((SELECT a.start + FROM peaks))`` — the wrapper-relation rewriter would walk + into the subquery and substitute wrapper aliases for refs + that only resolve in the subquery's own scope. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should fall back to the binned plan. + """ + # Arrange + query = ( + "SELECT SUM((SELECT a.start FROM peaks)) " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_fall_back_when_select_aggregate_contains_correlated_subquery( + self, + ): + """Test that a correlated subquery inside an aggregate argument falls back. + + Given: + ``SUM((SELECT a.start FROM genes WHERE genes.chrom = a.chrom))`` + in the SELECT list — same scope hazard as the uncorrelated + case. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should fall back to the binned plan. + """ + # Arrange + query = ( + "SELECT SUM((SELECT a.start FROM genes " + "WHERE genes.chrom = a.chrom)) " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" not in sql + + def test_transpile_should_resolve_projections_when_aliases_differ_in_case( + self, + ): + """Test that mixed-case aliases are normalized to match references. + + Given: + A query with ``peaks A`` / ``genes B`` (uppercase aliases) + and references via ``A.start`` / ``B.interval``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should emit a ``SET VARIABLE __giql_iejoin_`` block + (case-folded alias matching aligns with DuckDB's identifier + semantics, not Python's strict equality). + """ + # Arrange + query = ( + "SELECT A.start FROM peaks A " + "JOIN genes B ON A.interval INTERSECTS B.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + + def test_transpile_should_reject_extra_predicate_with_catalog_or_schema_qualified_column_when_dialect_is_duckdb( # noqa: E501 + self, + ): + """Test that a catalog/schema-qualified column in an extra predicate raises. + + Given: + A query whose extra predicate uses + ``mycat.myschema.a.score > 0`` — the alias-only rewriter + would leave the catalog/schema qualifier intact in the + inner subquery where it addresses a different relation + than intended. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` with a message naming the + catalog/schema qualifier. + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "AND mycat.myschema.a.score > 0" + ) + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + assert "catalog/schema qualifier" in str(excinfo.value) + + def test_transpile_should_engage_dialect_for_semi_join_with_intersects(self): + """Test that SEMI JOIN with INTERSECTS engages the dialect path. + + Given: + A query joining two tables with ``SEMI JOIN`` and an + INTERSECTS predicate. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should emit a ``SET VARIABLE __giql_iejoin_`` block. + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "SEMI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + + def test_transpile_should_emit_semi_keyword_when_join_is_semi(self): + """Test that SEMI JOIN's per-chromosome template uses the SEMI keyword. + + Given: + A SEMI JOIN + INTERSECTS query. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The emitted SQL should contain a ``SEMI JOIN`` substring + inside the per-chromosome template (not just ``JOIN``). + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "SEMI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SEMI JOIN" in sql + + def test_transpile_should_reject_right_side_projection_when_join_is_semi(self): + """Test that SEMI JOIN raises on ``b.col`` in the outer SELECT. + + Given: + A SEMI JOIN + INTERSECTS query whose outer SELECT projects + ``b.start`` — but SEMI returns left-side rows only. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` naming the left-only + constraint. + """ + # Arrange + query = ( + "SELECT b.start FROM peaks a " + "SEMI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + assert "left-side" in str(excinfo.value) + + def test_transpile_should_engage_dialect_for_anti_join_with_intersects(self): + """Test that ANTI JOIN with INTERSECTS engages the dialect path. + + Given: + A query joining two tables with ``ANTI JOIN`` and an + INTERSECTS predicate. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should emit a ``SET VARIABLE __giql_iejoin_`` block. + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + + def test_transpile_should_use_left_only_chrom_partition_when_join_is_anti(self): + """Test that ANTI JOIN's partition source is left-distinct only. + + Given: + An ANTI JOIN + INTERSECTS query. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + The emitted SET VARIABLE block should NOT contain + ``INTERSECT`` (the partition source is left-distinct only, + not the chromosome-INTERSECT used by INNER / SEMI). + """ + # Arrange + query = ( + "SELECT a.start FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + set_var_stmt = sql.split(";\n", 1)[0] + + # Assert + assert "INTERSECT" not in set_var_stmt + + def test_transpile_should_reject_right_side_projection_when_join_is_anti(self): + """Test that ANTI JOIN raises on ``b.col`` in the outer SELECT. + + Given: + An ANTI JOIN + INTERSECTS query whose outer SELECT projects + ``b.start``. + When: + ``transpile`` is called with ``dialect='duckdb'``. + Then: + It should raise ``ValueError`` naming the left-only + constraint. + """ + # Arrange + query = ( + "SELECT b.start FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks", "genes"], dialect="duckdb") + assert "left-side" in str(excinfo.value) + + +class TestTranspileDuckDBIEJoinExecution: + """End-to-end execution tests against in-memory DuckDB.""" + + def test_query_should_return_overlapping_pairs_when_executed_on_duckdb( + self, peaks_genes + ): + """Test the multi-chromosome inner-join overlap truth table. + + Given: + A peaks/genes fixture with a non-trivial truth table of + overlapping pairs across multiple chromosomes. + When: + The DuckDB-dialect SQL is executed against the connection. + Then: + It should return exactly the expected sorted set of + overlapping pairs. + """ + # Arrange + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(peaks_genes.execute(sql).fetchall()) + + # Assert + assert rows == sorted(_EXPECTED_OVERLAPS_PEAKS_GENES) + + def test_query_should_return_empty_set_when_no_chromosomes_intersect(self, conn): + """Test the empty-schema fallback when no chromosomes intersect. + + Given: + Two tables with disjoint chromosome sets. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return an empty result without raising (the + COALESCE empty-schema fallback fires). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p1", 0, "+")]) + _make_table(conn, "genes", [("chr2", 100, 200, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_match_binned_plan_results_when_executed_on_duckdb( + self, peaks_genes + ): + """Test that the binned plan and the IEJoin path agree on results. + + Given: + The ``peaks_genes`` fixture and a column-to-column + INTERSECTS INNER JOIN. + When: + The query is transpiled both with ``dialect=None`` (binned + plan) and with ``dialect='duckdb'`` (IEJoin path) and both + are executed. + Then: + Both should produce the same sorted set of overlapping + (peak, gene) rows. + """ + # Arrange + query = """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + binned_sql = transpile(query, tables=["peaks", "genes"]) + duckdb_sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + binned_rows = sorted(peaks_genes.execute(binned_sql).fetchall()) + duckdb_rows = sorted(peaks_genes.execute(duckdb_sql).fetchall()) + + # Assert + assert binned_rows == duckdb_rows + assert duckdb_rows == sorted(_EXPECTED_OVERLAPS_PEAKS_GENES) + + def test_query_should_handle_chrom_with_single_quote_in_name(self, conn): + """Test that single quotes in chrom names are escaped correctly. + + Given: + Intervals on a chromosome whose name contains a single + quote. + When: + The DuckDB-dialect SQL is executed. + Then: + It should escape the quote inside the dynamic SQL builder + and return the matching pair byte-for-byte. + """ + # Arrange + _make_table(conn, "peaks", [("chr'1", 100, 200, "p1", 0, "+")]) + _make_table(conn, "genes", [("chr'1", 150, 250, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [("chr'1", 100, 200, "chr'1", 150, 250)] + + def test_query_should_handle_one_based_closed_intervals(self, conn): + """Test that 1-based closed intervals report touching endpoints as overlapping. + + Given: + Tables declared as 1-based closed-closed intervals where + ``peaks[100, 200]`` and ``genes[200, 300]`` share endpoint + 200. + When: + The DuckDB-dialect SQL is executed. + Then: + It should report the pair as overlapping (closed-closed + shares endpoint 200). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p1", 0, "+")]) + _make_table(conn, "genes", [("chr1", 200, 300, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table("peaks", coordinate_system="1based", interval_type="closed"), + Table("genes", coordinate_system="1based", interval_type="closed"), + ], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert ("chr1", 100, 200, "chr1", 200, 300) in rows + + def test_query_should_not_report_non_touching_one_based_closed_as_overlapping( + self, conn + ): + """Test that strictly disjoint 1-based closed intervals do not match. + + Given: + Tables declared as 1-based closed-closed with non-touching + intervals (``peaks[100, 199]`` and ``genes[200, 300]``). + When: + The DuckDB-dialect SQL is executed. + Then: + It should report no overlap. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 199, "p1", 0, "+")]) + _make_table(conn, "genes", [("chr1", 200, 300, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table("peaks", coordinate_system="1based", interval_type="closed"), + Table("genes", coordinate_system="1based", interval_type="closed"), + ], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_handle_zero_based_closed_intervals(self, conn): + """Test that 0-based closed intervals match abutting endpoints. + + Given: + Tables declared as 0-based closed-closed intervals where + ``peaks[100, 200]`` abuts ``genes[200, 300]``. + When: + The DuckDB-dialect SQL is executed. + Then: + It should report the pair as overlapping (closed-closed + shares endpoint 200). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p1", 0, "+")]) + _make_table(conn, "genes", [("chr1", 200, 300, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table("peaks", coordinate_system="0based", interval_type="closed"), + Table("genes", coordinate_system="0based", interval_type="closed"), + ], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert ("chr1", 100, 200, "chr1", 200, 300) in rows + + def test_query_should_handle_implicit_cross_join_when_dialect_is_duckdb( + self, peaks_genes + ): + """Test that an implicit cross-join INTERSECTS produces the truth table. + + Given: + An implicit cross-join (``FROM peaks a, genes b WHERE + a.interval INTERSECTS b.interval``) over the multi-chromosome + ``peaks_genes`` fixture. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return exactly the expected sorted truth table of + overlapping pairs. + """ + # Arrange + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a, genes b + WHERE a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(peaks_genes.execute(sql).fetchall()) + + # Assert + assert rows == sorted(_EXPECTED_OVERLAPS_PEAKS_GENES) + + # --- DX-001..DX-011: behavioral truth tables ------------------------- + + def test_query_should_expand_qualified_star_when_dialect_is_duckdb(self, conn): + """Test that ``a.*, b.*`` expands to the genomic columns plus strand of each side. + + Given: + A ``SELECT a.*, b.*`` projection over an INTERSECTS join. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + Each row should carry chrom / start / end plus strand from + both sides (the default Table config declares strand_col), + and the result should contain the expected overlap pair. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p1", 7, "+")]) + _make_table(conn, "genes", [("chr1", 150, 250, "g1", 9, "-")]) + sql = transpile( + """ + SELECT a.*, b.* + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + result = conn.execute(sql) + rows = result.fetchall() + + # Assert + assert rows == [("chr1", 100, 200, "+", "chr1", 150, 250, "-")] + + def test_query_should_propagate_user_alias_on_qualified_column(self, conn): + """Test that a ``a.start AS s`` alias survives the IEJoin rewrite. + + Given: + A ``SELECT a.start AS s`` projection over an INTERSECTS + join. + When: + The query is transpiled with ``dialect='duckdb'`` and executed. + Then: + The result column should be named ``s`` and carry the value + of ``a.start`` for the matching pair. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 150, 250, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + result = conn.execute(sql) + column_names = [d[0] for d in result.description] + rows = result.fetchall() + + # Assert + assert column_names == ["s"] + assert rows == [(100,)] + + def test_query_should_treat_touching_half_open_intervals_as_non_overlapping( + self, conn + ): + """Test that half-open intervals that share an endpoint do not overlap. + + Given: + Two half-open tables where ``peaks[100, 200)`` ends exactly + where ``genes[200, 300)`` begins. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return an empty result (touching half-open + intervals do not overlap). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 200, 300, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table("peaks", coordinate_system="0based", interval_type="half_open"), + Table("genes", coordinate_system="0based", interval_type="half_open"), + ], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_handle_mixed_coordinate_systems(self, conn): + """Test semantic equivalence across mixed coordinate systems. + + Given: + ``peaks`` declared as 1-based closed (``[100, 200]``) and + ``genes`` declared as 0-based half-open (``[99, 200)``) + describing the same genomic span. + When: + The DuckDB-dialect SQL is executed. + Then: + It should report the pair as overlapping (both canonicalize + to the same 0-based half-open span). + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 99, 200, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table("peaks", coordinate_system="1based", interval_type="closed"), + Table("genes", coordinate_system="0based", interval_type="half_open"), + ], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [(100, 99)] + + def test_query_should_apply_one_based_offset_when_only_left_table_is_one_based( + self, conn + ): + """Test that the 1-based offset is applied per side, not globally. + + Given: + ``peaks`` declared as 1-based half-open and ``genes`` declared + as 0-based half-open, with values that overlap iff the offset + is applied to the peaks side + (``peaks[100, 101)`` and ``genes[99, 100)``). + When: + The DuckDB-dialect SQL is executed. + Then: + It should report the overlap; if the offset were skipped no + overlap would be reported. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 101, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 99, 100, "g", 0, "+")]) + sql_offset = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table( + "peaks", + coordinate_system="1based", + interval_type="half_open", + ), + Table( + "genes", + coordinate_system="0based", + interval_type="half_open", + ), + ], + dialect="duckdb", + ) + sql_no_offset = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table( + "peaks", + coordinate_system="0based", + interval_type="half_open", + ), + Table( + "genes", + coordinate_system="0based", + interval_type="half_open", + ), + ], + dialect="duckdb", + ) + + # Act + rows_offset = conn.execute(sql_offset).fetchall() + rows_no_offset = conn.execute(sql_no_offset).fetchall() + + # Assert + assert rows_offset == [(100, 99)] + assert rows_no_offset == [] + + def test_query_should_return_empty_when_left_table_is_empty(self, conn): + """Test the empty-left input case for the IEJoin path. + + Given: + ``peaks`` empty and ``genes`` populated with one row. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return an empty result without raising. + """ + # Arrange + _make_table(conn, "peaks", []) + _make_table(conn, "genes", [("chr1", 150, 250, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_return_empty_when_right_table_is_empty(self, conn): + """Test the empty-right input case for the IEJoin path. + + Given: + ``peaks`` populated with one row and ``genes`` empty. + When: + The DuckDB-dialect SQL is executed. + Then: + It should return an empty result without raising. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p1", 0, "+")]) + _make_table(conn, "genes", []) + sql = transpile( + """ + SELECT a.start AS a_s, b.start AS b_s + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [] + + def test_query_should_return_typed_empty_result_when_chromosome_intersection_is_empty( + self, conn + ): + """Test that the empty-schema fallback exposes correctly typed columns. + + Given: + Two tables whose chromosome sets are disjoint so the + chromosome-intersection ``string_agg`` is NULL. + When: + The DuckDB-dialect SQL is executed. + Then: + It should fetch an empty result whose column descriptions + match the source-table schema declared by ``_make_table`` + (chrom as VARCHAR, start/end as INTEGER). + """ + # Arrange + _make_table(conn, "peaks", [("chrA", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chrB", 100, 200, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + result = conn.execute(sql) + rows = result.fetchall() + type_by_name = {d[0]: str(d[1]) for d in result.description} + + # Assert + assert rows == [] + assert type_by_name["a_chrom"] == "VARCHAR" + assert type_by_name["b_chrom"] == "VARCHAR" + assert type_by_name["a_start"] == "INTEGER" + assert type_by_name["a_end"] == "INTEGER" + assert type_by_name["b_start"] == "INTEGER" + assert type_by_name["b_end"] == "INTEGER" + + def test_query_should_type_non_genomic_columns_in_empty_schema_fallback(self, conn): + """Test that non-genomic projected columns retain their real types when empty. + + Given: + Two tables whose chromosome sets are disjoint, projecting a + non-genomic column whose declared type is INTEGER (not VARCHAR + or BIGINT). + When: + The DuckDB-dialect SQL is executed. + Then: + It should fetch an empty result whose ``score`` column is typed + as INTEGER, matching the source table schema. + """ + # Arrange + conn.execute( + "CREATE TABLE peaks (chrom VARCHAR, start BIGINT, " + '"end" BIGINT, score INTEGER)' + ) + conn.execute( + "CREATE TABLE genes (chrom VARCHAR, start BIGINT, " + '"end" BIGINT, score INTEGER)' + ) + conn.execute("INSERT INTO peaks VALUES ('chrA', 100, 200, 7)") + conn.execute("INSERT INTO genes VALUES ('chrB', 100, 200, 9)") + sql = transpile( + """ + SELECT a.score AS a_score, b.score AS b_score + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + result = conn.execute(sql) + rows = result.fetchall() + type_by_name = {d[0]: str(d[1]) for d in result.description} + + # Assert + assert rows == [] + assert type_by_name["a_score"] == "INTEGER" + assert type_by_name["b_score"] == "INTEGER" + + def test_query_should_preserve_outer_join_semantics_when_dialect_is_duckdb( + self, conn + ): + """Test that LEFT JOIN INTERSECTS preserves unmatched left rows. + + Given: + A LEFT JOIN of ``peaks`` against ``genes`` on INTERSECTS, + with one peak that has no matching gene. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + The unmatched peak should appear in the result with NULL + gene columns (the IEJoin path falls back to the binned plan + for outer joins). + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p_match", 0, "+"), + ("chr1", 5000, 6000, "p_lonely", 0, "+"), + ], + ) + _make_table(conn, "genes", [("chr1", 150, 250, "g1", 0, "+")]) + sql = transpile( + """ + SELECT a.start AS a_start, b.start AS b_start + FROM peaks a + LEFT JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall(), key=lambda r: r[0]) + + # Assert + assert rows == [(100, 150), (5000, None)] + + def test_query_should_preserve_extra_join_predicate_when_dialect_is_duckdb( + self, conn + ): + """Test that an extra non-INTERSECTS predicate is honored end-to-end. + + Given: + A query whose JOIN ON carries + ``a.interval INTERSECTS b.interval AND a.score > b.score``, + with input rows where one pair overlaps and satisfies the + score predicate and another overlaps but violates it. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + Only the pair that overlaps and satisfies the extra + predicate should be returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p_high", 100, "+"), + ("chr1", 300, 400, "p_low", 1, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g_low", 5, "+"), + ("chr1", 350, 450, "g_high", 50, "+"), + ], + ) + sql = transpile( + """ + SELECT a.start AS a_start, b.start AS b_start + FROM peaks a + JOIN genes b + ON a.interval INTERSECTS b.interval + AND a.score > b.score + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert rows == [(100, 150)] + + def test_query_should_return_correct_rows_for_literal_intersects_when_dialect_is_duckdb( + self, conn + ): + """Test that literal-range INTERSECTS produces the same rows as default. + + Given: + A WHERE literal-range INTERSECTS query over ``peaks``. + When: + The query is transpiled with ``dialect=None`` and with + ``dialect='duckdb'`` and both are executed. + Then: + Both should return the same sorted rows. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 0, "+"), + ("chr1", 1500, 1700, "p2", 0, "+"), + ("chr1", 5000, 6000, "p3", 0, "+"), + ], + ) + query = "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'" + + # Act + rows_default = sorted( + conn.execute(transpile(query, tables=["peaks"])).fetchall() + ) + rows_duckdb = sorted( + conn.execute(transpile(query, tables=["peaks"], dialect="duckdb")).fetchall() + ) + + # Assert + assert rows_default == rows_duckdb + assert rows_duckdb == [("chr1", 1500, 1700, "p2", 0, "+")] + + def test_query_should_partition_on_custom_chrom_col_when_table_uses_custom_columns( + self, conn + ): + """Test that the IEJoin partition uses the configured chrom column. + + Given: + DuckDB tables with ``chromosome``/``start_pos``/``end_pos`` + custom column names and rows on multiple chromosomes (only + one of which has overlapping intervals). + When: + The query is transpiled with the matching ``Table`` configs + and ``dialect='duckdb'`` and executed. + Then: + It should return only the rows where the custom-named + chromosome column matches and the ranges overlap. + """ + # Arrange + conn.execute( + "CREATE TABLE peaks (chromosome VARCHAR, start_pos INTEGER, end_pos INTEGER)" + ) + conn.execute( + "INSERT INTO peaks VALUES " + "('chr1', 100, 200), " + "('chr2', 100, 200), " + "('chr3', 100, 200)" + ) + conn.execute( + "CREATE TABLE genes (chromosome VARCHAR, start_pos INTEGER, end_pos INTEGER)" + ) + conn.execute( + "INSERT INTO genes VALUES " + "('chr1', 150, 250), " + "('chr2', 1000, 2000), " + "('chr3', 5000, 6000)" + ) + sql = transpile( + """ + SELECT a.chromosome AS a_chrom, a.start_pos AS a_start, + b.start_pos AS b_start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=[ + Table( + "peaks", + chrom_col="chromosome", + start_col="start_pos", + end_col="end_pos", + ), + Table( + "genes", + chrom_col="chromosome", + start_col="start_pos", + end_col="end_pos", + ), + ], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert rows == [("chr1", 100, 150)] + + def test_query_should_preserve_row_multiplicity_for_duplicate_input_rows(self, conn): + """Test that duplicate input rows surface as duplicate output rows. + + Given: + Two identical ``peaks`` rows that each overlap the same single + ``genes`` row, projecting genomic columns from both sides. + When: + The DuckDB-dialect SQL is executed. + Then: + The result should contain *two* identical overlap rows, not + one — a regression detector for a rewrite that silently + collapses duplicates via an unintended ``DISTINCT``. (The + property-based oracle below sorts a multiset, so this is the + canonical-example pin for the same invariant.) + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p_dup", 1, "+"), + ("chr1", 100, 200, "p_dup", 1, "+"), + ], + ) + _make_table( + conn, + "genes", + [("chr1", 150, 250, "g1", 10, "-")], + ) + sql = transpile( + "SELECT a.chrom AS c, a.start AS s, a.end AS e " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert rows == [("chr1", 100, 200), ("chr1", 100, 200)] + + def test_query_should_order_lexicographically_when_order_by_has_two_columns( + self, conn + ): + """Test that ORDER BY on two keys orders lexicographically. + + Given: + Multiple overlap rows where the first ORDER BY key has ties + and the second key discriminates among them. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + Rows should come back in lexicographic order on the two + keys. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 100, 200, "p2", 2, "+"), + ("chr1", 300, 400, "p3", 3, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g_a", 10, "+"), + ("chr1", 350, 450, "g_b", 20, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s, b.score AS gs FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start ASC, b.score DESC", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # Two chr1 peaks at start=100 overlap g_a (score 10); chr1 peak at + # start=300 overlaps g_b (score 20). Within the start=100 ties, + # b.score DESC is a no-op (only one matching gene), so the keyed + # order is: (100, 10), (100, 10), then (300, 20). + assert rows == [(100, 10), (100, 10), (300, 20)] + + def test_query_should_skip_first_m_and_return_remainder_when_offset_only(self, conn): + """Test that OFFSET past the first ordered row returns the rest. + + Given: + Three ordered overlap rows on chr1. + When: + The query is transpiled with ``LIMIT 5 OFFSET 1`` and + executed. + Then: + The trailing two rows (after skipping the first) are + returned, in order. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 300, 400, "p2", 2, "+"), + ("chr1", 500, 600, "p3", 3, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ("chr1", 550, 650, "g3", 3, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "ORDER BY a.start LIMIT 5 OFFSET 1", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(300,), (500,)] + + def test_query_should_dedup_on_tuple_when_select_distinct_has_multiple_columns( + self, conn + ): + """Test that ``SELECT DISTINCT`` over a multi-column list dedups on tuples. + + Given: + Multiple overlap rows that produce duplicate + ``(a.chrom, b.chrom)`` tuples but distinct ``a.start`` + values. + When: + The query is transpiled with ``SELECT DISTINCT a.chrom AS + ca, b.chrom AS cb`` and executed. + Then: + The result should collapse to the set of distinct tuples, + not deduplicate on either column alone. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 300, 400, "p2", 2, "+"), + ("chr2", 100, 200, "p3", 3, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ("chr2", 150, 250, "g3", 3, "+"), + ], + ) + sql = transpile( + "SELECT DISTINCT a.chrom AS ca, b.chrom AS cb FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = set(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # Two chr1 join rows collapse to one ('chr1','chr1') tuple; chr2 + # contributes one ('chr2','chr2') tuple. Total: 2 distinct tuples. + assert rows == {("chr1", "chr1"), ("chr2", "chr2")} + + def test_query_should_filter_cross_side_predicate_in_where_when_dialect_is_duckdb( + self, conn + ): + """Test that a cross-side predicate in WHERE filters rows correctly. + + Given: + Three overlapping pairs on chr1 where only those whose + peak's ``start`` is strictly less than the gene's ``start`` + satisfy the cross-side WHERE predicate. (Variant of the + ON-side cross-predicate test — exercises the WHERE-extras + extraction path rather than the JOIN-ON extras path.) + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + Only the pair satisfying the cross-side WHERE predicate is + returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ( + "chr1", + 100, + 200, + "p_lt", + 1, + "+", + ), # a.start (100) < b.start (150) → keep + ( + "chr1", + 300, + 400, + "p_eq", + 2, + "+", + ), # a.start (300) < b.start (350) → keep + ( + "chr1", + 500, + 600, + "p_gt", + 3, + "+", + ), # a.start (500) > b.start (490) → drop + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr1", 350, 450, "g2", 0, "+"), + ("chr1", 490, 590, "g3", 0, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS a_s, b.start AS b_s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE a.start < b.start", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100, 150), (300, 350)] + + def test_query_should_filter_by_between_predicate_when_dialect_is_duckdb(self, conn): + """Test that ``BETWEEN`` filters overlap rows. + + Given: + Three overlap pairs with varied ``a.score``; the predicate + ``a.score BETWEEN 10 AND 50`` excludes the row at score=1 + and the row at score=99. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + Only the score=20 row should be returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p_low", 1, "+"), + ("chr1", 300, 400, "p_mid", 20, "+"), + ("chr1", 500, 600, "p_high", 99, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr1", 350, 450, "g2", 0, "+"), + ("chr1", 550, 650, "g3", 0, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "AND a.score BETWEEN 10 AND 50", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(300,)] + + def test_query_should_honor_is_null_predicate_when_dialect_is_duckdb(self, conn): + """Test that ``WHERE a.score IS NULL`` filters overlap rows. + + Given: + Two overlap pairs on chr1 — one with NULL ``a.score`` and + one with a non-NULL score. + When: + The query is transpiled with ``WHERE a.score IS NULL`` and + executed. + Then: + Only the NULL-score row should be returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p_null", None, "+"), + ("chr1", 300, 400, "p_set", 99, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr1", 350, 450, "g2", 0, "+"), + ], + ) + sql = transpile( + "SELECT a.start AS s FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "WHERE a.score IS NULL", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100,)] + + def test_query_should_rewrite_user_aliases_p_g_to_inner_a_b_when_dialect_is_duckdb( + self, conn + ): + """Test that user aliases other than ``a``/``b`` get rewritten at runtime. + + Given: + A join written as ``FROM peaks p JOIN genes g`` (user + aliases ``p`` and ``g``, not ``a`` and ``b``) with an extra + cross-side ON predicate ``p.score > g.score``. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + DuckDB should not raise an alias-resolution error — the + dialect rewrote ``p`` / ``g`` to the inner subquery's + hardcoded ``a`` / ``b`` aliases — and the filter should + return only the pair satisfying the cross-side predicate. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 100, "+"), + ("chr1", 300, 400, "p2", 5, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g_low", 10, "+"), + ("chr1", 350, 450, "g_high", 50, "+"), + ], + ) + sql = transpile( + "SELECT p.start AS p_s, g.start AS g_s FROM peaks p " + "JOIN genes g ON p.interval INTERSECTS g.interval " + "AND p.score > g.score", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [(100, 150)] + + def test_query_should_count_non_null_column_values_per_group_when_dialect_is_duckdb( + self, conn + ): + """Test that ``COUNT(a.name)`` counts only non-NULL names per group. + + Given: + Three overlap rows on chr1 — two with non-NULL ``a.name`` + and one with NULL — plus one overlap on chr2 with non-NULL + ``a.name``. + When: + The query is transpiled with ``COUNT(a.name) GROUP BY + a.chrom`` and executed. + Then: + The chr1 count should be 2 (excluding the NULL), and the + chr2 count should be 1. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 110, 210, None, 2, "+"), + ("chr1", 120, 220, "p3", 3, "+"), + ("chr2", 100, 200, "p4", 4, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr2", 150, 250, "g2", 2, "-"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, COUNT(a.name) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", 2), ("chr2", 1)] + + def test_query_should_group_by_multiple_columns_when_dialect_is_duckdb(self, conn): + """Test that multi-column GROUP BY groups by tuple. + + Given: + Overlap rows whose ``(a.chrom, a.strand)`` tuples are + heterogeneous: two ``(chr1, +)`` overlaps, one ``(chr1, -)`` + overlap. + When: + The query is transpiled with ``GROUP BY a.chrom, a.strand`` + and executed. + Then: + One row per distinct ``(chrom, strand)`` tuple should be + returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 110, 210, "p2", 2, "+"), + ("chr1", 300, 400, "p3", 3, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 350, 450, "g2", 2, "+"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, a.strand AS s, COUNT(*) AS n FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom, a.strand", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr1", "+", 2), ("chr1", "-", 1)] + + def test_query_should_filter_groups_when_having_combines_two_conditions(self, conn): + """Test that ``HAVING`` with two AND-ed aggregate conditions filters groups. + + Given: + Three chromosomes; chr1 has many low-score overlaps, chr2 + has one high-score overlap, chr3 has many high-score + overlaps. Only chr3 satisfies both + ``COUNT(*) > 1 AND SUM(a.score) > 100``. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + Only chr3 should be returned. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + # chr1: 3 overlaps, all score 1 → COUNT > 1, SUM = 3 + ("chr1", 100, 200, "p1", 1, "+"), + ("chr1", 110, 210, "p2", 1, "+"), + ("chr1", 120, 220, "p3", 1, "+"), + # chr2: 1 overlap, score 99 → COUNT == 1 (fails) + ("chr2", 100, 200, "p4", 99, "+"), + # chr3: 2 overlaps, scores 60 each → COUNT > 1, SUM = 120 + ("chr3", 100, 200, "p5", 60, "+"), + ("chr3", 110, 210, "p6", 60, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr2", 150, 250, "g2", 0, "+"), + ("chr3", 150, 250, "g3", 0, "+"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, COUNT(*) AS n, SUM(a.score) AS s " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom HAVING COUNT(*) > 1 AND SUM(a.score) > 100", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert rows == [("chr3", 2, 120)] + + def test_query_should_compose_all_tier1_features_in_single_pipeline_when_dialect_is_duckdb( + self, conn + ): + """Test the kitchen-sink composition of every Tier 1 feature. + + Given: + A query combining an extra ON predicate (``a.score > 20``), + an extra WHERE predicate (``b.score < 50``), multi-column + ``GROUP BY``, a non-trivial ``HAVING`` (``SUM(a.score) > + 40`` — actually filters one of the groups out), ``ORDER BY + n DESC``, and ``LIMIT 10`` — over data crafted so the + counts after both filters are 2/1/0 on chr1/chr2/chr3 but + chr2's surviving sum is below the HAVING threshold. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + Only chr1 survives — its 2-overlap SUM(a.score) of 70 beats + the threshold; chr2's single 30 fails it; chr3 was already + dropped by the ON predicate. LIMIT 10 has no effect (one + group survives). Aggregates SUM and COUNT(*) coexist and + the ORDER BY references the user-aliased COUNT alias. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + # chr1: two peaks both pass a.score > 20; sum = 70 + ("chr1", 100, 200, "p1", 30, "+"), + ("chr1", 110, 210, "p2", 40, "+"), + # chr2: one peak passes a.score > 20; sum = 30 (fails HAVING) + ("chr2", 100, 200, "p3", 30, "-"), + # chr3: peak fails a.score > 20 + ("chr3", 100, 200, "p4", 10, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + # chr1 gene passes b.score < 50 (=10) + ("chr1", 150, 250, "g1", 10, "+"), + # chr2 gene passes b.score < 50 (=20) + ("chr2", 150, 250, "g2", 20, "-"), + # chr3 gene would pass but no overlap-survivor exists + ("chr3", 150, 250, "g3", 5, "+"), + ], + ) + sql = transpile( + "SELECT a.chrom AS c, COUNT(*) AS n, SUM(a.score) AS s " + "FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval " + "AND a.score > 20 " + "WHERE b.score < 50 " + "GROUP BY a.chrom HAVING SUM(a.score) > 40 " + "ORDER BY n DESC LIMIT 10", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + # chr1: 2 overlaps, sum(a.score)=70 → passes HAVING + # chr2: 1 overlap, sum(a.score)=30 → fails HAVING + # chr3: 0 overlaps (peak filtered by ON) → no group + assert rows == [("chr1", 2, 70)] + + @pytest.mark.parametrize( + ("coordinate_system", "interval_type"), + [ + ("0based", "half_open"), + ("0based", "closed"), + ("1based", "half_open"), + ("1based", "closed"), + ], + ) + def test_query_should_match_binned_plan_for_coordinate_system_combinations( + self, conn, coordinate_system, interval_type + ): + """Test cross-plan equivalence across all 4 coord-system × interval-type combos. + + Given: + A fixed 6-row truth table of peaks/genes with overlaps that + depend on how endpoints are interpreted. + When: + The same logical query is transpiled twice — once with + ``dialect=None`` (binned plan) and once with + ``dialect="duckdb"`` — under each of the 4 + ``(coordinate_system, interval_type)`` combinations. + Then: + Both plans should return the same sorted multiset for every + combination, proving canonicalization composes with the + dialect rewrite consistently. + """ + # Arrange — fixture uses strict (non-touching) overlaps so the + # overlap set is identical across all 4 coord-system × interval + # combos. The test then asserts that canonicalization composes + # with the dialect rewrite without changing which pairs match. + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 0, "+"), + ("chr1", 300, 400, "p2", 0, "+"), + ("chr1", 700, 800, "p3", 0, "+"), + ("chr2", 100, 200, "p4", 0, "-"), + ("chr2", 500, 600, "p5", 0, "-"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 0, "+"), + ("chr1", 350, 450, "g2", 0, "+"), + ("chr2", 50, 180, "g3", 0, "-"), + ("chr2", 550, 700, "g4", 0, "-"), + ], + ) + tables = [ + Table( + "peaks", + coordinate_system=coordinate_system, + interval_type=interval_type, + ), + Table( + "genes", + coordinate_system=coordinate_system, + interval_type=interval_type, + ), + ] + query = ( + "SELECT a.chrom AS c, a.start AS s " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval" + ) + sql_default = transpile(query, tables=tables) + sql_duckdb = transpile(query, tables=tables, dialect="duckdb") + # Python-native ground truth. The fixture uses strict (non- + # touching) overlaps, so the matching set is identical across + # all 4 coord-system × interval combos — that's the property + # the test is pinning. Asserting against this expected list + # (not just `rows_default == rows_duckdb`) catches a regression + # in which both plans share the same bug. + expected = [ + ("chr1", 100), # peaks[100,200) ∩ genes[150,250) + ("chr1", 300), # peaks[300,400) ∩ genes[350,450) + ("chr2", 100), # peaks[100,200) ∩ genes[50,180) + ("chr2", 500), # peaks[500,600) ∩ genes[550,700) + ] + + # Act + rows_default = sorted(conn.execute(sql_default).fetchall()) + rows_duckdb = sorted(conn.execute(sql_duckdb).fetchall()) + + # Assert + assert rows_duckdb == expected + assert rows_default == rows_duckdb + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=200), + st.integers(min_value=1, max_value=50), + ), + min_size=0, + max_size=8, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=200), + st.integers(min_value=1, max_value=50), + ), + min_size=0, + max_size=8, + ), + ) + def test_query_should_match_python_native_overlap_for_random_inputs( + self, conn, peaks, genes + ): + """Test the IEJoin path against a Python-native overlap reference. + + Given: + A Hypothesis-generated pair of small interval lists drawn + from a tiny chromosome alphabet and small int starts/lengths + (half-open coordinates on both sides). + When: + The DuckDB-dialect SQL is executed and compared against a + Python-native overlap reference + (``a.end > b.start AND b.end > a.start AND a.chrom == b.chrom``). + Then: + The set of overlapping ``(chrom, start, end, chrom, start, + end)`` tuples should be identical between the two methods. + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if peak_rows: + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if gene_rows: + conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + sql_rows = sorted(conn.execute(sql).fetchall()) + expected = _python_overlap(peak_rows, gene_rows) + + # Assert: sorted-multiset equality — a `set` here would collapse + # duplicate input rows and silently hide multiplicity bugs. + assert sql_rows == expected + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + ), + min_size=0, + max_size=6, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + ), + min_size=0, + max_size=6, + ), + ) + def test_query_should_match_binned_plan_for_random_inputs(self, conn, peaks, genes): + """Test that the binned plan and the IEJoin path agree on random inputs. + + Given: + A Hypothesis-generated pair of small interval lists. + When: + The same logical query is transpiled both with + ``dialect=None`` and with ``dialect='duckdb'`` and executed. + Then: + Both plans should return the same multiset of overlapping + rows. + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if peak_rows: + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if gene_rows: + conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + query = """ + SELECT a.chrom AS a_chrom, a.start AS a_start, a.end AS a_end, + b.chrom AS b_chrom, b.start AS b_start, b.end AS b_end + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = sorted(conn.execute(sql_default).fetchall()) + rows_duckdb = sorted(conn.execute(sql_duckdb).fetchall()) + + # Assert + assert rows_default == rows_duckdb + + @settings( + max_examples=30, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + st.integers(min_value=0, max_value=50), + ), + min_size=1, + max_size=6, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + st.integers(min_value=0, max_value=50), + ), + min_size=1, + max_size=6, + ), + score_threshold=st.integers(min_value=0, max_value=50), + ) + def test_query_should_match_binned_plan_when_where_group_count_order_combined( + self, conn, peaks, genes, score_threshold + ): + """Test cross-plan equivalence with extras, GROUP BY, and ORDER BY. + + Given: + Hypothesis-generated peak / gene interval lists carrying + scores, and a random ``score_threshold``. + When: + The same logical query — combining a WHERE extra predicate, + a GROUP BY, a COUNT(*) aggregate, and ORDER BY — is + transpiled both with ``dialect=None`` and ``dialect='duckdb'`` + and executed. + Then: + Both plans should return the same multiset of grouped rows. + """ + # Arrange + peak_rows = [(c, s, s + length, score) for (c, s, length, score) in peaks] + gene_rows = [(c, s, s + length, score) for (c, s, length, score) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + conn.executemany("INSERT INTO genes VALUES (?, ?, ?, ?)", gene_rows) + query = ( + "SELECT a.chrom AS c, COUNT(*) AS n " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval " + f"WHERE a.score >= {score_threshold} " + "GROUP BY a.chrom " + "ORDER BY a.chrom" + ) + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = conn.execute(sql_default).fetchall() + rows_duckdb = conn.execute(sql_duckdb).fetchall() + + # Assert + assert rows_default == rows_duckdb + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + chrom_name=st.text( + alphabet=st.characters( + min_codepoint=0x20, + max_codepoint=0x7E, + blacklist_characters=["\x00", "%", "?"], + ), + min_size=1, + max_size=10, + ), + ) + def test_query_should_round_trip_arbitrary_chrom_names_under_dialect( + self, conn, chrom_name + ): + """Test that arbitrary printable chrom names round-trip the IEJoin path. + + Given: + A Hypothesis-drawn printable-ASCII chromosome name (length + 1..10, excluding NUL, ``%``, ``?``) that may include single + quotes, double quotes, or backslashes, seeded as the chrom + value of a single overlapping pair. + When: + The DuckDB-dialect SQL is generated and executed. + Then: + The pair should be returned exactly once and the chrom value + should round-trip byte-for-byte. + """ + # Arrange + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + conn.execute("INSERT INTO peaks VALUES (?, ?, ?)", (chrom_name, 100, 200)) + conn.execute("INSERT INTO genes VALUES (?, ?, ?)", (chrom_name, 150, 250)) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = conn.execute(sql).fetchall() + + # Assert + assert rows == [(chrom_name, 100)] + + @settings( + max_examples=30, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + # Nullable score so IS NULL / IS NOT NULL aren't + # tautological against an always-populated column. + st.one_of(st.none(), st.integers(min_value=0, max_value=50)), + ), + min_size=1, + max_size=5, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + st.one_of(st.none(), st.integers(min_value=0, max_value=50)), + ), + min_size=1, + max_size=5, + ), + predicate_kind=st.sampled_from( + ["cmp", "between", "in_list", "cross_side", "is_null"] + ), + threshold=st.integers(min_value=0, max_value=50), + ) + def test_query_should_match_binned_plan_for_random_extra_where_predicates( + self, conn, peaks, genes, predicate_kind, threshold + ): + """Test cross-plan equivalence under random WHERE extras. + + Given: + Hypothesis-generated peak / gene interval lists with + (possibly NULL) score columns, plus a sampled predicate + kind from ``{cmp, between, in_list, cross_side, is_null}``. + When: + The composed query is transpiled under both ``dialect=None`` + and ``dialect="duckdb"`` and executed. + Then: + Both plans should return the same sorted multiset. + """ + # Arrange + peak_rows = [(c, s, s + length, score) for (c, s, length, score) in peaks] + gene_rows = [(c, s, s + length, score) for (c, s, length, score) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + conn.executemany("INSERT INTO genes VALUES (?, ?, ?, ?)", gene_rows) + where_clauses = { + "cmp": f"WHERE a.score >= {threshold}", + "between": f"WHERE a.score BETWEEN {threshold} AND {threshold + 20}", + "in_list": "WHERE a.score IN (0, 10, 20, 30, 40, 50)", + "cross_side": "WHERE a.score >= b.score", + "is_null": "WHERE a.score IS NOT NULL", + } + where = where_clauses[predicate_kind] + query = ( + "SELECT a.chrom AS c, a.start AS s " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval " + f"{where}" + ) + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = sorted(conn.execute(sql_default).fetchall()) + rows_duckdb = sorted(conn.execute(sql_duckdb).fetchall()) + + # Assert + assert rows_default == rows_duckdb + + @settings( + max_examples=30, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + st.integers(min_value=0, max_value=50), + ), + min_size=1, + max_size=5, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + st.integers(min_value=0, max_value=50), + ), + min_size=1, + max_size=5, + ), + aggregate_kind=st.sampled_from( + [ + "count_star", + "sum_a_score", + "min_b_start", + "max_b_end", + "count_distinct_b_chrom", + ] + ), + ) + def test_query_should_match_binned_plan_for_random_group_by_aggregates( + self, conn, peaks, genes, aggregate_kind + ): + """Test cross-plan equivalence for GROUP BY + random aggregate. + + Given: + Hypothesis-generated peak / gene interval lists with score + columns, plus a sampled aggregate from a small whitelist. + When: + A query with ``GROUP BY a.chrom`` and the chosen aggregate + is transpiled under both ``dialect=None`` and + ``dialect="duckdb"`` and executed. + Then: + Both plans should return the same row list (sorted by chrom + for stable comparison; ``AVG`` excluded from this PBT to + keep equality strict — covered by an example test instead). + """ + # Arrange + peak_rows = [(c, s, s + length, score) for (c, s, length, score) in peaks] + gene_rows = [(c, s, s + length, score) for (c, s, length, score) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + conn.executemany("INSERT INTO genes VALUES (?, ?, ?, ?)", gene_rows) + aggregates = { + "count_star": "COUNT(*)", + "sum_a_score": "SUM(a.score)", + "min_b_start": "MIN(b.start)", + "max_b_end": "MAX(b.end)", + "count_distinct_b_chrom": "COUNT(DISTINCT b.chrom)", + } + agg = aggregates[aggregate_kind] + query = ( + f"SELECT a.chrom AS c, {agg} AS v " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval " + "GROUP BY a.chrom ORDER BY a.chrom" + ) + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = conn.execute(sql_default).fetchall() + rows_duckdb = conn.execute(sql_duckdb).fetchall() + + # Assert — ORDER BY makes positional comparison meaningful. + assert rows_default == rows_duckdb + + @settings( + # Full-composite has the largest input space (extras + GROUP BY + # + HAVING + ORDER BY + LIMIT + OFFSET), so we run more examples. + max_examples=60, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + # Allow ``min_size=0`` on peaks so empty-input cross-plan + # equivalence is exercised in combination with the full Tier 1 + # stack (the simpler PBTs use min_size=1). + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + st.integers(min_value=0, max_value=50), + ), + min_size=0, + max_size=6, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + st.integers(min_value=0, max_value=50), + ), + min_size=0, + max_size=6, + ), + score_threshold=st.integers(min_value=0, max_value=50), + having_min=st.integers(min_value=0, max_value=5), + # Include LIMIT 0 so the trivial-truncation edge case is + # exercised against both plans. + limit=st.integers(min_value=0, max_value=10), + ) + def test_query_should_match_binned_plan_for_full_composite_query( + self, conn, peaks, genes, score_threshold, having_min, limit + ): + """Test cross-plan equivalence under the full Tier 1 composition. + + Given: + Hypothesis-generated peaks/genes with random thresholds for + an extra WHERE predicate, a HAVING minimum count, and a + LIMIT. + When: + A query exercising extra ON + extra WHERE + GROUP BY + + HAVING + ORDER BY + LIMIT is transpiled under both + ``dialect=None`` and ``dialect="duckdb"`` and executed. + Then: + Both plans should return the same ordered row list (``==``, + because ``ORDER BY`` makes order part of the contract). + """ + # Arrange + peak_rows = [(c, s, s + length, score) for (c, s, length, score) in peaks] + gene_rows = [(c, s, s + length, score) for (c, s, length, score) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + # DuckDB's executemany rejects empty parameter lists; skip the + # INSERT step when the strategy generates no rows. + if peak_rows: + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, ' + '"end" INTEGER, score INTEGER)' + ) + if gene_rows: + conn.executemany("INSERT INTO genes VALUES (?, ?, ?, ?)", gene_rows) + query = ( + "SELECT a.chrom AS c, COUNT(*) AS n " + "FROM peaks a JOIN genes b " + "ON a.interval INTERSECTS b.interval AND a.score >= b.score " + f"WHERE a.score >= {score_threshold} " + "GROUP BY a.chrom " + f"HAVING COUNT(*) >= {having_min} " + "ORDER BY a.chrom " + f"LIMIT {limit}" + ) + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = conn.execute(sql_default).fetchall() + rows_duckdb = conn.execute(sql_duckdb).fetchall() + + # Assert + assert rows_default == rows_duckdb + + def test_query_should_match_inner_join_result_when_using_join_targets_chrom_only( + self, peaks_genes + ): + """Test that USING(chrom) returns the same rows as plain INNER JOIN. + + Given: + The shared peaks_genes fixture and two queries — one using + ``JOIN ... ON a.interval INTERSECTS b.interval`` and another + using ``JOIN ... USING (chrom) WHERE a.interval INTERSECTS + b.interval`` — both with ``dialect='duckdb'``. + When: + Both are executed. + Then: + They should return identical sorted row sets, because + USING(chrom) is redundant with the dialect's per-chromosome + partition. + """ + # Arrange + plain_sql = transpile( + "SELECT a.chrom, a.start, a.end " + "FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + using_sql = transpile( + "SELECT a.chrom, a.start, a.end " + "FROM peaks a JOIN genes b USING (chrom) " + "WHERE a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + plain_rows = sorted(peaks_genes.execute(plain_sql).fetchall()) + using_rows = sorted(peaks_genes.execute(using_sql).fetchall()) + + # Assert + assert plain_rows == using_rows + + def test_query_should_match_binned_plan_for_mixed_case_aliases(self, conn): + """Test that mixed-case aliases under the dialect match the binned plan. + + Given: + The shared peaks_genes fixture loaded onto ``conn`` and a + query using uppercase aliases (``peaks A`` / ``genes B``). + When: + The same logical query is transpiled both with + ``dialect=None`` and ``dialect='duckdb'`` and executed. + Then: + Both plans should return the same multiset of rows. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 100, 200, "p1", 10, "+"), + ("chr1", 500, 600, "p2", 20, "+"), + ], + ) + _make_table( + conn, + "genes", + [ + ("chr1", 150, 250, "g1", 1, "+"), + ("chr1", 500, 600, "g2", 2, "+"), + ], + ) + query = ( + "SELECT A.chrom, A.start, B.start " + "FROM peaks A JOIN genes B " + "ON A.interval INTERSECTS B.interval" + ) + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = sorted(conn.execute(sql_default).fetchall()) + rows_duckdb = sorted(conn.execute(sql_duckdb).fetchall()) + + # Assert + assert rows_default == rows_duckdb + + def test_query_should_preserve_rows_from_left_only_chromosomes_when_join_is_anti(self, conn): + """Test that ANTI JOIN preserves left-only chromosomes (C1b regression). + + Given: + Peaks on chr1 and chr3; genes on chr1 only. Under ANTI JOIN + with INTERSECTS, chr3 peaks must be returned (they have no + overlapping right-side row), and chr1 peaks that don't + overlap any gene must also be returned. + When: + The query is transpiled with ``dialect='duckdb'`` and + executed. + Then: + It should return the chr3 peak and any chr1 peaks with no + overlap — the chrom-INTERSECT partition would have excluded + chr3 entirely, so this is the critical regression for + C1b's left-distinct partition swap. + """ + # Arrange + _make_table( + conn, + "peaks", + [ + ("chr1", 10, 20, "p1", 0, "+"), # no overlap with chr1 gene + ("chr1", 100, 200, "p2", 0, "+"), # overlaps chr1 gene + ("chr3", 1, 1000, "p3", 0, "+"), # chr3 — no genes there + ], + ) + _make_table( + conn, + "genes", + [("chr1", 50, 150, "g1", 0, "+")], + ) + sql = transpile( + "SELECT a.chrom, a.start, a.end " + "FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert rows == sorted([("chr1", 10, 20), ("chr3", 1, 1000)]) + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=200), + st.integers(min_value=1, max_value=50), + ), + min_size=0, + max_size=8, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=200), + st.integers(min_value=1, max_value=50), + ), + min_size=0, + max_size=8, + ), + ) + def test_query_should_match_python_native_semi_overlap_for_random_inputs( + self, conn, peaks, genes + ): + """Test SEMI JOIN against a Python-native semi-overlap reference. + + Given: + A Hypothesis-generated pair of small interval lists drawn + from a tiny chromosome alphabet. + When: + A ``SEMI JOIN ... ON a.interval INTERSECTS b.interval`` is + transpiled with ``dialect='duckdb'`` and executed. + Then: + The set of returned ``(chrom, start, end)`` tuples should + equal the Python reference (distinct left rows that have + at least one overlapping right row). + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if peak_rows: + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if gene_rows: + conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + sql = transpile( + "SELECT a.chrom, a.start, a.end " + "FROM peaks a " + "SEMI JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + sql_rows = sorted(set(conn.execute(sql).fetchall())) + expected = _python_semi_overlap(peak_rows, gene_rows) + + # Assert + assert sql_rows == expected + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + ), + min_size=0, + max_size=6, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2"]), + st.integers(min_value=0, max_value=100), + st.integers(min_value=1, max_value=30), + ), + min_size=0, + max_size=6, + ), + ) + def test_query_should_match_binned_plan_for_semi_join_random_inputs( + self, conn, peaks, genes + ): + """Test that SEMI JOIN under the dialect matches the binned plan. + + Given: + A Hypothesis-generated pair of small interval lists. + When: + The same SEMI JOIN INTERSECTS query is transpiled with + ``dialect=None`` (binned) and ``dialect='duckdb'`` and + executed. + Then: + Both plans should return the same multiset of distinct + left rows. + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if peak_rows: + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if gene_rows: + conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + query = ( + "SELECT a.chrom, a.start, a.end " + "FROM peaks a " + "SEMI JOIN genes b ON a.interval INTERSECTS b.interval" + ) + sql_default = transpile(query, tables=["peaks", "genes"]) + sql_duckdb = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Act + rows_default = sorted(set(conn.execute(sql_default).fetchall())) + rows_duckdb = sorted(set(conn.execute(sql_duckdb).fetchall())) + + # Assert + assert rows_default == rows_duckdb + + @settings( + max_examples=25, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], + ) + @given( + peaks=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=200), + st.integers(min_value=1, max_value=50), + ), + min_size=0, + max_size=8, + ), + genes=st.lists( + st.tuples( + st.sampled_from(["chr1", "chr2", "chr3"]), + st.integers(min_value=0, max_value=200), + st.integers(min_value=1, max_value=50), + ), + min_size=0, + max_size=8, + ), + ) + def test_query_should_match_python_native_anti_overlap_for_random_inputs( + self, conn, peaks, genes + ): + """Test ANTI JOIN against a Python-native anti-overlap reference. + + Given: + A Hypothesis-generated pair of small interval lists. + When: + An ``ANTI JOIN ... ON a.interval INTERSECTS b.interval`` is + transpiled with ``dialect='duckdb'`` and executed. + Then: + The set of returned ``(chrom, start, end)`` tuples should + equal the Python reference (distinct left rows with no + overlapping right row, including rows on chromosomes + absent from the right table). + """ + # Arrange + peak_rows = [(c, s, s + length) for (c, s, length) in peaks] + gene_rows = [(c, s, s + length) for (c, s, length) in genes] + conn.execute("DROP TABLE IF EXISTS peaks") + conn.execute("DROP TABLE IF EXISTS genes") + conn.execute( + 'CREATE TABLE peaks (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if peak_rows: + conn.executemany("INSERT INTO peaks VALUES (?, ?, ?)", peak_rows) + conn.execute( + 'CREATE TABLE genes (chrom VARCHAR, "start" INTEGER, "end" INTEGER)' + ) + if gene_rows: + conn.executemany("INSERT INTO genes VALUES (?, ?, ?)", gene_rows) + sql = transpile( + "SELECT a.chrom, a.start, a.end " + "FROM peaks a " + "ANTI JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Act + sql_rows = sorted(set(conn.execute(sql).fetchall())) + expected = _python_anti_overlap(peak_rows, gene_rows) + + # Assert + assert sql_rows == expected + + # NOTE: a `test_query_should_match_binned_plan_for_anti_join_random_inputs` + # would also be desirable, but the binned plan currently returns the + # WRONG result for ANTI JOIN when the right table is empty (it drops + # all left rows, where ANTI semantics demands that every left row + # passes). The dialect IS correct for that case — covered by + # ``test_query_should_preserve_rows_from_left_only_chromosomes_when_join_is_anti`` and + # ``test_query_should_match_python_native_anti_overlap_for_random_inputs`` + # — so cross-plan parity for ANTI is not a useful contract until the + # binned plan's ANTI handling is fixed in a follow-up. + + +class TestTranspileDuckDBIEJoinKwargs: + """Tests for the ``dialect`` / ``intersects_bin_size`` kwarg surface. + + Covers the ``@overload`` pair on :func:`giql.transpile.transpile`, the + mutual-exclusivity validation between ``dialect`` and + ``intersects_bin_size``, the unknown-dialect error, and one + behavioral regression — running ``intersects_bin_size`` with + ``dialect`` omitted — that verifies adding the dialect kwarg did not + perturb the binned plan. + """ + + def test_transpile_should_match_default_when_dialect_none_is_explicit(self): + """Test that ``dialect=None`` is equivalent to omitting the kwarg. + + Given: + A column-to-column INTERSECTS JOIN. + When: + ``transpile`` is called once with ``dialect=None`` explicit + and once with the kwarg omitted entirely. + Then: + Both calls should produce byte-identical SQL output. + """ + # Arrange + query = """ + SELECT a.chrom, a.start, b.start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act + sql_omitted = transpile(query, tables=["peaks", "genes"]) + sql_explicit_none = transpile(query, tables=["peaks", "genes"], dialect=None) + + # Assert + assert sql_omitted == sql_explicit_none + + def test_transpile_should_accept_intersects_bin_size_when_dialect_is_omitted( + self, conn + ): + """Test that ``intersects_bin_size`` works when ``dialect`` is omitted. + + Given: + A column-to-column INTERSECTS JOIN. + When: + ``transpile`` is called with ``intersects_bin_size=50000`` and + no ``dialect`` and the resulting SQL is executed. + Then: + It should return the correct overlapping rows. + """ + # Arrange + _make_table(conn, "peaks", [("chr1", 100, 200, "p", 0, "+")]) + _make_table(conn, "genes", [("chr1", 150, 250, "g", 0, "+")]) + sql = transpile( + """ + SELECT a.chrom AS a_chrom, a.start AS a_start, b.start AS b_start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + intersects_bin_size=50000, + ) + + # Act + rows = sorted(conn.execute(sql).fetchall()) + + # Assert + assert rows == [("chr1", 100, 150)] + + def test_transpile_should_echo_offending_value_in_unknown_dialect_error(self): + """Test that the unknown-dialect error names the offending value. + + Given: + An unknown dialect string ``'postgres'``. + When: + ``transpile`` is called. + Then: + It should raise ``ValueError`` whose message contains both + ``Unknown dialect`` and the literal ``'postgres'``. + """ + # Arrange + query = "SELECT * FROM peaks" + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile(query, tables=["peaks"], dialect="postgres") + message = str(excinfo.value) + assert "Unknown dialect" in message + assert "'postgres'" in message + + def test_transpile_should_mention_both_kwargs_in_mutex_error_message(self): + """Test that the mutex error names both ``intersects_bin_size`` and ``duckdb``. + + Given: + ``dialect='duckdb'`` together with a non-None + ``intersects_bin_size``. + When: + ``transpile`` is called. + Then: + It should raise ``ValueError`` whose message mentions both + ``intersects_bin_size`` and ``duckdb``. + """ + # Arrange + query = """ + SELECT a.chrom, a.start, b.start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """ + + # Act & assert + with pytest.raises(ValueError) as excinfo: + transpile( + query, + tables=["peaks", "genes"], + dialect="duckdb", + intersects_bin_size=50000, + ) + message = str(excinfo.value) + assert "intersects_bin_size" in message + assert "duckdb" in message From e25c177ac35389554cc4c19e21ab8e17a110d0b8 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:03:25 -0400 Subject: [PATCH 030/142] docs: Document DuckDB IEJoin dialect in performance guide Adds a "DuckDB IEJoin dialect" subsection covering the opt-in via dialect="duckdb", the supported INNER / SEMI / ANTI shapes including the absorbed decorations (ORDER BY / LIMIT / OFFSET / DISTINCT, GROUP BY / HAVING with aggregates, extra ON / WHERE predicates), the automatic fallbacks (each unsupported shape enumerated), the mutual exclusivity with intersects_bin_size, the empty-schema typing caveat, and the full enumeration of projection-shape rejections that raise rather than fall back. The wording uses the range-join family terminology rather than naming IE_JOIN specifically, since DuckDB may pick PIECEWISE_MERGE_JOIN for single-inequality cases. --- docs/transpilation/performance.rst | 246 +++++++++++++++++++++++++++++ 1 file changed, 246 insertions(+) diff --git a/docs/transpilation/performance.rst b/docs/transpilation/performance.rst index 3815cd5..881f266 100644 --- a/docs/transpilation/performance.rst +++ b/docs/transpilation/performance.rst @@ -301,6 +301,252 @@ Analyze query execution plans by running EXPLAIN on the transpiled SQL: Backend-Specific Tips --------------------- +DuckDB IEJoin Dialect +~~~~~~~~~~~~~~~~~~~~~ + +For column-to-column ``INTERSECTS`` joins (INNER, SEMI, or ANTI) on +DuckDB, the ``dialect="duckdb"`` opt-in transpiles the join into a +per-chromosome dynamic-SQL pattern. Each per-chromosome subquery contains +only inequality predicates, which DuckDB plans through its range-join +family (``IE_JOIN`` or ``PIECEWISE_MERGE_JOIN``) — avoiding the +row-inflation and deduplication cost of the default binned equi-join plan +(the ``dialect=None`` path, controlled by ``intersects_bin_size``, which +expands each interval into ``UNNEST``-generated bins and deduplicates +the resulting matches). + +.. code-block:: python + + from giql import transpile + + sql = transpile( + """ + SELECT a.chrom, a.start, b.start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + # sql is a two-statement script — see the next example for execution. + +The output is a multi-statement string of the form:: + + SET VARIABLE __giql_iejoin_ = COALESCE((... string_agg per chromosome ...), ''); + SELECT ... FROM query(getvariable('__giql_iejoin_')) AS __giql_iejoin_wrapper + +The ```` is a per-call ``uuid4().hex`` (128 bits) suffix so the +``SET VARIABLE`` name is collision-resistant even when outputs from many +``transpile()`` calls are interleaved in a single DuckDB session +(session variables are global session state). The outer SELECT's +wrapper-relation alias is constant (``__giql_iejoin_wrapper``) because +it isn't user-visible and doesn't need to vary per call. + +Inside the per-chromosome string-builder, chromosome names are emitted +via ``replace(chrom, '''', '''''')`` wrapped in single quotes, so +chromosome identifiers containing literal single quotes interpolate +safely into the dynamic SQL. + +Because the output is a two-statement script that depends on session +state, execute it through a single DuckDB connection's ``.execute()`` — +DuckDB ≥1.4 (the version pinned by GIQL's development and CI +environment) accepts multi-statement strings in one call and returns +the result of the final statement. SQLAlchemy's ``text()``, +``pandas.read_sql_query``, and similar wrappers that split or rewrite +the string may drop the ``SET VARIABLE`` and produce empty or NULL +results. + +.. code-block:: python + + import duckdb + from giql import transpile + + conn = duckdb.connect() + conn.execute("CREATE TABLE peaks (chrom VARCHAR, \"start\" INTEGER, \"end\" INTEGER)") + conn.execute("CREATE TABLE genes (chrom VARCHAR, \"start\" INTEGER, \"end\" INTEGER)") + sql = transpile( + """ + SELECT a.chrom, a.start, b.start + FROM peaks a + JOIN genes b ON a.interval INTERSECTS b.interval + """, + tables=["peaks", "genes"], + dialect="duckdb", + ) + rows = conn.execute(sql).fetchall() + +The dialect rewrites the whole query, so the supported shape is +``SELECT FROM {INNER|SEMI|ANTI} +JOIN {ON|WHERE} `` — the +JOIN and the column-to-column ``INTERSECTS`` are both required +(literal-range ``INTERSECTS`` against a single table falls through to +the default predicate generator). + +**Join variants.** ``INNER JOIN`` is the default. ``SEMI JOIN`` returns +distinct left rows that have at least one overlapping match; ``ANTI +JOIN`` returns left rows with no overlapping match. Both restrict the +outer SELECT to left-side projections — any reference to the right +side (``b.col`` / ``b.*`` / aggregate over ``b.*``) raises +``ValueError``. ANTI uses a left-distinct chromosome partition rather +than the chromosome INTERSECT used by INNER / SEMI, so left rows on +chromosomes absent from the right table are preserved. + +DuckDB plans INNER through the IE_JOIN / PIECEWISE_MERGE_JOIN +sort-merge family. SEMI and ANTI with inequality predicates plan +through ``BLOCKWISE_NL_JOIN`` instead — still avoiding the UNNEST +row-inflation of the binned plan, but not the IE_JOIN sort-merge fast +path. Expect substantial speedups vs. the binned plan for SEMI / ANTI +where the chromosome partition already filters most pairs; INNER gets +the largest speedup. + +On top of the core shape the dialect also absorbs several common +decorations: + +- **Outer modifiers.** ``ORDER BY`` / ``LIMIT`` / ``OFFSET`` / + ``DISTINCT`` on the outer query are appended to the outer SELECT; + column references in ``ORDER BY`` are rewritten to the wrapper + relation's inner aliases. +- **Aggregates.** ``GROUP BY`` / ``HAVING`` with aggregate functions + (``COUNT``, ``SUM``, ``MIN``, ``MAX``, ``AVG``, ``COUNT(DISTINCT + a.col)``, ...) are appended to the outer SELECT. Aggregate arguments + must be table-qualified (``COUNT(*)`` and ``COUNT(a.col)`` are both + accepted). +- **Extra JOIN / WHERE predicates.** Additional non-INTERSECTS + predicates ANDed onto the join ON or WHERE (e.g. ``a.score > + b.score`` or ``WHERE a.score > 100``) are inlined into each + per-chromosome subquery's ON, so DuckDB filters them inside each + IEJoin candidate set. **Limitations:** the dialect peels extra + predicates only across top-level ``AND`` connectives, so predicates + that wrap the ``INTERSECTS`` in ``OR`` / ``NOT`` / parentheses still + fall back. Predicates whose AND-tree residual contains a subquery, + aggregate, or window function fall back too (DuckDB forbids window + functions inside ``JOIN ON``, and subqueries / aggregates inside an + IEJoin candidate set would either break the planner or produce + semantically wrong results). If you hit the + WHERE-INTERSECTS-plus-extra-JOIN-ON-predicate shape described in + `#94 `_ on the default + ``dialect=None`` path, ``dialect="duckdb"`` is one workaround — the + dialect inlines extras directly into each per-chromosome subquery + and is unaffected by that bug. +- **Star projections (``a.*`` / ``b.*``).** Expanded to the genomic + columns declared by the corresponding :class:`Table` config (chrom + / start / end, plus strand when set). This narrows the result schema + relative to the binned plan, which delegates star expansion to + DuckDB and projects every base-table column. Users with additional + non-genomic columns should list them explicitly. + +The dialect splits unsupported shapes into two buckets. Soft-fallback +shapes route to the binned plan automatically and return correct +results; within those shapes the ``dialect`` kwarg is safe to set +without risk of silent incorrectness. Hard-error shapes (enumerated +further below) raise ``ValueError`` at transpile time — the dialect +deliberately refuses them rather than silently producing the wrong +SQL. + +The soft-fallback shapes are: + +- **Outer joins.** ``LEFT`` / ``RIGHT`` / ``FULL`` ``JOIN ... ON ... + INTERSECTS ...`` falls back to the binned plan. The same applies + when the join keeps its side modifier and the ``INTERSECTS`` lives in + the top-level ``WHERE`` (e.g. ``LEFT JOIN ... ON TRUE WHERE + a.interval INTERSECTS b.interval``). +- **NATURAL joins.** ``NATURAL JOIN`` falls back because the dialect + cannot enumerate shared columns at transpile time (only the + registered ``chrom`` / ``start`` / ``end`` / ``strand`` columns are + known). +- **USING joins.** Single-column ``USING()`` admits (the + per-chromosome partition is exactly the equi-join). Multi-column + ``USING`` and ``USING()`` fall back; inline support + is a documented follow-up. +- **Multiple INTERSECTS predicates.** Queries with more than one + column-to-column ``INTERSECTS`` (anywhere in the AST, including + subqueries) fall back. +- **INTERSECTS references unrecognized join sides.** A column-to-column + INTERSECTS whose operands don't reference the FROM table's alias, + or where the second operand's alias doesn't resolve to a registered + JOIN target, falls back to the binned plan. +- **Self-joins.** Joining a table to itself falls back. +- **OR / NOT / paren-wrapped extra predicates.** See the Extra JOIN / + WHERE predicates note above. +- **Subquery, aggregate, or window-function extra predicates.** + Predicates whose AND-tree residual contains a subquery, an aggregate + function, or a window function (``ROW_NUMBER() OVER (...)`` etc.) + fall back rather than inline into the per-chromosome subquery's + ``ON`` (DuckDB forbids window functions in ``ON`` clauses, and + aggregates / subqueries inside an IEJoin candidate set would either + break the planner or produce semantically wrong results). +- **Subquery inside GROUP BY / HAVING / ORDER BY.** A modifier clause + containing a nested subquery (including ``EXISTS (SELECT ...)``) + falls back to the binned plan, because the dialect's modifier-ref + rewriter is not scope-aware and would corrupt the subquery's + column scope. Subqueries hidden inside aggregate arguments in the + SELECT list fall back for the same reason. +- **Top-level WITH clauses.** A query with a leading ``WITH`` falls + back so the CTE survives. +- **More than two tables.** Any extra FROM-clause table (auxiliary + comma-join or non-INTERSECTS ``JOIN``) falls back. +- **Non-base-table operands.** A subquery, CTE, or GIQL table-function + (e.g. ``DISJOIN(genes)``) used as a join operand falls back. +- **DISTINCT ON (...).** Only plain ``DISTINCT`` is absorbed; the + ``DISTINCT ON`` variant falls back. + +Query-shape ``ValueError`` raises (the dialect deliberately refuses +these rather than silently miscompile): + +- **Bare** ``SELECT *`` **and unqualified columns.** The dialect path + needs to know which side each output column comes from. Use ``a.*``, + ``b.*``, ``a.col``, or ``a.col AS x``. +- **Expression-form projections.** ``a.start + 1`` and other non-column + expressions (other than aggregates) raise. Project the underlying + column and compute in a wrapping query around the dialect's output, + or omit ``dialect="duckdb"`` to use the binned plan. +- **Unknown table qualifier.** ``c.col`` when the only sides are ``a`` + and ``b``. +- **Unknown alias in GROUP BY / HAVING / ORDER BY.** A modifier-clause + column qualified with a table alias outside the join's two sides + (``c.col``) raises with a ``cannot resolve … in `` message + naming the expected aliases. +- **Right-side reference in SEMI / ANTI joins.** ``b.col`` / ``b.*`` / + aggregate over ``b.*`` in the outer SELECT, GROUP BY, HAVING, ORDER + BY, or aggregate argument under ``SEMI JOIN`` / ``ANTI JOIN`` raises; + SEMI / ANTI return left-side columns only. +- **Aggregate of unqualified column.** ``SUM(score)`` raises; use + ``SUM(a.score)`` or ``SUM(b.score)``. +- **Star projection with a user alias.** ``a.* AS x`` raises (most + engines reject star-with-alias outright); either drop the alias and + use ``a.*``, or list the columns explicitly with per-column aliases. +- **Window aggregates and FILTER clauses.** ``SUM(a.score) OVER + (...)`` and ``SUM(a.score) FILTER (WHERE ...)`` raise with a + dedicated error; rewrite the projection or omit ``dialect="duckdb"``. +- **Aggregates inside expressions.** ``COUNT(*) * 2`` and similar + arithmetic-over-aggregate projections raise; project the aggregate + on its own and do arithmetic in a wrapping query. +- **Scalar subqueries in the SELECT list.** ``(SELECT SUM(x) FROM + other_table) AS s`` raises; rewrite without the subquery or omit + ``dialect="duckdb"``. +- **Catalog/schema-qualified extra predicates.** ``mycat.myschema.a.col`` + in an extra predicate raises; the alias rewriter would leave the + catalog/schema qualifier intact in the inner subquery, where it + references a different relation than intended. +- **Unqualified or unknown-aliased extra predicates.** ``WHERE strand + = '+'`` (unqualified) or ``WHERE c.score > 0`` (alias outside the + join's two sides) raise with an actionable message naming the + expected aliases. The binned plan would either silently bind the + column wrong or defer the error to the downstream engine, so the + dialect surfaces the user mistake at transpile time instead. + +Argument-validation ``ValueError`` raises (these fire before any AST +inspection and are independent of the query shape): + +- **Mutually exclusive with** ``intersects_bin_size``. The two + parameters cannot be combined; pass one or the other. +- **Unknown dialect string.** Anything other than ``"duckdb"`` or + ``None`` raises with the offending value echoed. + +Literal-range ``INTERSECTS`` (e.g. ``WHERE interval INTERSECTS +'chr1:1000-2000'``) is single-table and has no column-to-column join +to rewrite, so the dialect declines and the standard range-predicate +emission handles it identically to the ``dialect=None`` path. + DuckDB Optimizations ~~~~~~~~~~~~~~~~~~~~ From 7b6da7590a5962f0b50603bd375fbd6fea9ee6b0 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:03:30 -0400 Subject: [PATCH 031/142] docs: Mention DuckDB IEJoin dialect opt-in in README A short README pointer so users browsing the project page see the dialect="duckdb" knob and follow through to the performance guide for the supported shapes and the full fallback / rejection enumeration. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fccf9f7..20e97c1 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,8 @@ print(sql) The transpiled SQL can be executed with fast genome-unaware databases or in-memory analytic engines like DuckDB. +For column-to-column `INTERSECTS` joins (INNER, SEMI, or ANTI) targeting DuckDB, pass `dialect="duckdb"` to opt into a per-chromosome IEJoin plan that avoids the row inflation of the default binned equi-join. See [DuckDB IEJoin Dialect](https://giql.readthedocs.io/en/latest/transpilation/performance.html#duckdb-iejoin-dialect) for the supported shapes and fallback rules. + You can also use [oxbow](https://oxbow.readthedocs.io) to efficiently stream specialized genomics formats into DuckDB. ```python From 8491707a1b2484c77340d3f41e0391b0cf0eeac4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 May 2026 11:03:36 -0400 Subject: [PATCH 032/142] chore: Refresh demo.ipynb kernel metadata to Python 3.13.3 Updates the notebook's recorded kernel display name and language version to match the interpreter the demo is now developed against. No code-cell content changes. --- demo.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/demo.ipynb b/demo.ipynb index 1d136ae..5d5573e 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -224,7 +224,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -3755,7 +3755,7 @@ ], "metadata": { "kernelspec": { - "display_name": "giql", + "display_name": "giql (3.13.3)", "language": "python", "name": "python3" }, @@ -3769,7 +3769,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.13.3" } }, "nbformat": 4, From 3093f47ddaae3c50e97e7bd91506d8b0b567a639 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 9 Jun 2026 17:29:35 -0400 Subject: [PATCH 033/142] feat: Add ResolveOperatorRefs normalization pass scaffolding Introduce the first normalization pass of the AST pipeline between the transformer chain and the generator. The pass walks sqlglot scopes leaves-first, resolves each operator's table-shaped reference slots against in-query CTEs and registered tables, and attaches a frozen ResolvedRef under the node's meta mapping. Per-operator SlotSpec descriptors declare each slot's accepted shapes on the expression classes, and a closing validation boundary asserts the attached metadata is well-formed. This lands with zero behavior change: the generator keeps its existing resolver paths and ignores the new metadata, and unresolvable slots are deferred to the generator so its diagnostics fire verbatim. Later migration steps port each operator emitter to consume the resolved metadata. --- src/giql/expressions.py | 128 ++++++++++-- src/giql/resolver.py | 449 ++++++++++++++++++++++++++++++++++++++++ src/giql/transpile.py | 8 + 3 files changed, 572 insertions(+), 13 deletions(-) create mode 100644 src/giql/resolver.py diff --git a/src/giql/expressions.py b/src/giql/expressions.py index fd97a62..4364fce 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -1,11 +1,79 @@ """Custom AST expression nodes for genomic operations. This module defines custom SQLGlot expression nodes for GIQL spatial operators. + +It also declares the per-operator :class:`SlotSpec` descriptors that the +``ResolveOperatorRefs`` normalization pass (:mod:`giql.resolver`) consumes. Each +operator class carries a ``GIQL_SLOTS`` tuple naming the slots the operator +resolves (``this`` / ``reference`` / ``expression`` / ``ranges``) together with +the AST shapes that slot accepts. Keeping the acceptance rules declarative on the +expression class — rather than duplicated across resolver code — is the structural +change epic #114 is built around. """ +from dataclasses import dataclass +from typing import Literal + from sqlglot import exp from sqlglot.errors import ParseError +# The shapes an operator slot may accept. The first three form the +# "registered table / CTE / subquery" trichotomy that ``sqlglot``'s scope +# machinery distinguishes natively and that :class:`giql.resolver.ResolvedRef` +# models; the remaining three describe the column / literal / implicit-outer +# forms used by NEAREST, DISTANCE, and the spatial predicates. +SlotShape = Literal[ + "registered_table", + "cte", + "subquery", + "literal_range", + "column", + "implicit_outer", +] + +# The subset of slot shapes that resolve to a table-shaped relation and are +# therefore representable as a :class:`giql.resolver.ResolvedRef`. A slot whose +# accepted shapes are a subset of these is a "reference slot" that the +# ``ResolveOperatorRefs`` pass resolves and attaches metadata for. +TABLE_SHAPES: frozenset[SlotShape] = frozenset({"registered_table", "cte", "subquery"}) + + +@dataclass(frozen=True, slots=True) +class SlotSpec: + """Declarative description of one resolvable operator slot. + + Parameters + ---------- + arg : str + The :attr:`arg_types` key naming the slot on the operator expression + (e.g. ``"this"``, ``"reference"``, ``"expression"``, ``"ranges"``). + accepts : frozenset[SlotShape] + The AST shapes the slot accepts. + required : bool + Whether the slot must be present on a well-formed operator. + is_sequence : bool + Whether the slot holds a list of expressions (e.g. the ``ranges`` slot + of :class:`SpatialSetPredicate`) rather than a single expression. + """ + + arg: str + accepts: frozenset[SlotShape] + required: bool = False + is_sequence: bool = False + + @property + def is_ref_slot(self) -> bool: + """Whether this slot resolves to a table-shaped :class:`ResolvedRef`. + + A reference slot is one whose accepted shapes are all drawn from the + registered-table / CTE / subquery trichotomy. These are the slots for + which the ``ResolveOperatorRefs`` pass resolves and attaches a + :class:`giql.resolver.ResolvedRef`; column / literal / implicit + slots are declared but their resolution is deferred to later epic #114 + migration steps. + """ + return bool(self.accepts) and self.accepts <= TABLE_SHAPES + class GenomicRange(exp.Expression): """Represents a parsed genomic range. @@ -37,7 +105,10 @@ class Intersects(SpatialPredicate): Example: column INTERSECTS 'chr1:1000-2000' """ - pass + GIQL_SLOTS = ( + SlotSpec("this", frozenset({"column"}), required=True), + SlotSpec("expression", frozenset({"literal_range", "column"}), required=True), + ) class Contains(SpatialPredicate): @@ -46,7 +117,10 @@ class Contains(SpatialPredicate): Example: column CONTAINS 'chr1:1500' """ - pass + GIQL_SLOTS = ( + SlotSpec("this", frozenset({"column"}), required=True), + SlotSpec("expression", frozenset({"literal_range", "column"}), required=True), + ) class Within(SpatialPredicate): @@ -55,7 +129,10 @@ class Within(SpatialPredicate): Example: column WITHIN 'chr1:1000-5000' """ - pass + GIQL_SLOTS = ( + SlotSpec("this", frozenset({"column"}), required=True), + SlotSpec("expression", frozenset({"literal_range", "column"}), required=True), + ) class SpatialSetPredicate(exp.Expression): @@ -73,6 +150,16 @@ class SpatialSetPredicate(exp.Expression): "ranges": True, } + GIQL_SLOTS = ( + SlotSpec("this", frozenset({"column"}), required=True), + SlotSpec( + "ranges", + frozenset({"literal_range"}), + required=True, + is_sequence=True, + ), + ) + def _split_named_and_positional(args): """Separate named parameters (:= and =>) from positional arguments.""" @@ -114,8 +201,7 @@ def from_arg_list(cls, args): kwargs["distance"] = positional_args[1] if "this" not in kwargs: raise ParseError( - "CLUSTER requires a genomic interval column as its first " - "argument." + "CLUSTER requires a genomic interval column as its first argument." ) return cls(**kwargs) @@ -147,8 +233,7 @@ def from_arg_list(cls, args): kwargs["distance"] = positional_args[1] if "this" not in kwargs: raise ParseError( - "MERGE requires a genomic interval column as its first " - "argument." + "MERGE requires a genomic interval column as its first argument." ) return cls(**kwargs) @@ -174,6 +259,11 @@ class GIQLDistance(exp.Func): "signed": False, # Optional: boolean for directional distance } + GIQL_SLOTS = ( + SlotSpec("this", frozenset({"column"}), required=True), + SlotSpec("expression", frozenset({"column"}), required=True), + ) + @classmethod def from_arg_list(cls, args): kwargs, positional_args = _split_named_and_positional(args) @@ -206,15 +296,21 @@ class GIQLNearest(exp.Func): "signed": False, # Optional: directional distance } + GIQL_SLOTS = ( + SlotSpec("this", frozenset({"registered_table"}), required=True), + SlotSpec( + "reference", + frozenset({"literal_range", "column", "implicit_outer"}), + ), + ) + @classmethod def from_arg_list(cls, args): kwargs, positional_args = _split_named_and_positional(args) if len(positional_args) >= 1: kwargs["this"] = positional_args[0] if "this" not in kwargs: - raise ParseError( - "NEAREST requires a target table as its first argument." - ) + raise ParseError("NEAREST requires a target table as its first argument.") return cls(**kwargs) @@ -240,6 +336,14 @@ class GIQLDisjoin(exp.Func): "reference": False, # Optional: reference table/CTE name or subquery } + GIQL_SLOTS = ( + SlotSpec("this", frozenset({"registered_table"}), required=True), + SlotSpec( + "reference", + frozenset({"registered_table", "cte", "subquery"}), + ), + ) + @classmethod def from_arg_list(cls, args): kwargs, positional_args = _split_named_and_positional(args) @@ -258,7 +362,5 @@ def from_arg_list(cls, args): if len(positional_args) >= 1: kwargs["this"] = positional_args[0] if "this" not in kwargs: - raise ParseError( - "DISJOIN requires a target table as its first argument." - ) + raise ParseError("DISJOIN requires a target table as its first argument.") return cls(**kwargs) diff --git a/src/giql/resolver.py b/src/giql/resolver.py new file mode 100644 index 0000000..07b3a91 --- /dev/null +++ b/src/giql/resolver.py @@ -0,0 +1,449 @@ +"""Pass 1 of the GIQL normalization pipeline: ``ResolveOperatorRefs``. + +This module implements the first normalization pass described in epic #114. It +runs between the transformer chain and the generator and attaches resolution +metadata to every GIQL operator node so that later epic steps can turn the +generator into a pure formatter. + +The pass is built on ``sqlglot``'s scope machinery +(:func:`sqlglot.optimizer.scope.traverse_scope`), which yields scopes +leaves-first so CTE and derived-table sources are registered before their +consumers, and whose ``cte_sources`` natively distinguishes an in-query CTE from +a registered table — exactly the registered-table / CTE / subquery trichotomy +each operator reference slot must resolve against. + +For every GIQL operator node the pass resolves each declared *reference slot* +(see :class:`giql.expressions.SlotSpec`) against the enclosing scope plus the +registered :class:`giql.table.Tables` container and attaches a frozen +:class:`ResolvedRef` to the node through a single namespaced +``node.meta["giql"]`` key — ``sqlglot``'s established metadata channel, which is +deep-copied by ``Expression.copy()`` so resolution survives tree copies. The +pass closes with a validation boundary (:func:`validate_operator_refs`) that +asserts every operator slot carries well-formed resolution metadata, mirroring +``sqlglot``'s ``validate_qualify_columns`` and Spark's ``CheckAnalysis``. + +Scope note (epic #114, step 1) +------------------------------ +This is *scaffolding*: it lands with **zero behavior change**. The generator +still uses its existing resolver paths and ignores everything attached here. The +resolution semantics computed for the table-shaped reference slots mirror the +generator's existing ``_resolve_target_table`` / ``_resolve_disjoin_reference`` +/ ``_enclosing_cte_names`` behavior exactly. + +Two consequences of the zero-behavior-change constraint shape the +implementation: + +* The pass never raises a *user-facing* diagnostic. When a slot cannot be + resolved (unregistered target, unknown reference name, reserved ``__giql_dj_`` + prefix, unsupported reference shape) the pass simply leaves that slot + unresolved and the generator raises its existing error exactly as before. + Promoting these into GIQL-level diagnostics is a later epic step. +* Only *reference slots* — slots whose accepted shapes are the table + trichotomy (DISJOIN ``this``/``reference`` and NEAREST ``this``) — are + resolved to a :class:`ResolvedRef` here. The column / literal / + implicit-outer slots of NEAREST, DISTANCE, and the spatial predicates are + declared on the expression classes but their resolution metadata type is + designed by the per-operator port issues (#118, #119, #120). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from dataclasses import replace +from typing import Literal + +from sqlglot import exp +from sqlglot.optimizer.scope import Scope +from sqlglot.optimizer.scope import traverse_scope + +from giql.constants import DEFAULT_CHROM_COL +from giql.constants import DEFAULT_END_COL +from giql.constants import DEFAULT_START_COL +from giql.expressions import Contains +from giql.expressions import GIQLDisjoin +from giql.expressions import GIQLDistance +from giql.expressions import GIQLNearest +from giql.expressions import Intersects +from giql.expressions import SlotSpec +from giql.expressions import SpatialSetPredicate +from giql.expressions import Within +from giql.table import Table +from giql.table import Tables + +__all__ = [ + "META_KEY", + "RefKind", + "ResolvedRef", + "OperatorResolution", + "ResolutionError", + "resolve_operator_refs", + "validate_operator_refs", +] + +#: The single namespaced key under which resolution metadata is stored on a +#: node's ``.meta`` mapping. All GIQL metadata lives under this one key so the +#: channel stays a single, greppable namespace. +META_KEY = "giql" + +#: The kinds a resolved reference slot can take — the registered-table / CTE / +#: subquery trichotomy that ``sqlglot``'s ``scope.sources`` distinguishes. +RefKind = Literal["registered_table", "cte", "subquery"] + +#: Canonical default genomic column names. A CTE or subquery reference is +#: assumed (in the existing generator and here) to expose canonical +#: ``chrom`` / ``start`` / ``end`` columns; validating that contract is a later +#: epic step. +_DEFAULT_COLS: tuple[str, str, str] = ( + DEFAULT_CHROM_COL, + DEFAULT_START_COL, + DEFAULT_END_COL, +) + +#: The GIQL operator expression classes the pass inspects. +_OPERATORS: tuple[type[exp.Expression], ...] = ( + GIQLDisjoin, + GIQLNearest, + GIQLDistance, + Intersects, + Contains, + Within, + SpatialSetPredicate, +) + + +class ResolutionError(Exception): + """Raised when attached resolution metadata is malformed. + + This is an internal invariant check fired by :func:`validate_operator_refs`, + not a user-facing GIQL diagnostic. It indicates a bug in the resolver pass — + a slot carrying metadata that does not satisfy its declared + :class:`~giql.expressions.SlotSpec` — rather than an invalid user query. + """ + + +@dataclass(frozen=True, slots=True) +class ResolvedRef: + """Resolved metadata for one table-shaped operator reference slot. + + Attributes + ---------- + kind : RefKind + Whether the slot resolved to a registered table, an in-query CTE, or a + subquery. + name : str | None + The relation name for a registered-table or CTE reference; ``None`` for + an (anonymous) subquery reference. + cols : tuple[str, str, str] + The ``(chrom, start, end)`` physical column names. For a registered + table these come from its :class:`~giql.table.Table` config; for a CTE + or subquery they are the canonical defaults the relation is assumed to + expose. + table : Table | None + The :class:`~giql.table.Table` config backing a registered-table + reference (carrying its coordinate system); ``None`` for CTE and + subquery references, which are assumed canonical. + coverage_skippable : bool + Whether this reference is provably the same relation as the operator's + target set — i.e. a self-reference — which lets DISJOIN omit its + coverage ``EXISTS`` filter. Always ``False`` for non-reference (target) + slots. + """ + + kind: RefKind + name: str | None + cols: tuple[str, str, str] + table: Table | None + coverage_skippable: bool + + +@dataclass(frozen=True, slots=True) +class OperatorResolution: + """Resolution metadata attached to a single GIQL operator node. + + Attributes + ---------- + operator : str + The operator expression class name (e.g. ``"GIQLDisjoin"``). + slots : dict[str, ResolvedRef] + Mapping from a slot's :attr:`~giql.expressions.SlotSpec.arg` key to its + resolved :class:`ResolvedRef`. Only successfully resolved reference + slots appear; a slot left out was either not a reference slot or could + not be resolved (and the generator will raise its existing error). + """ + + operator: str + slots: dict[str, ResolvedRef] + + def slot(self, arg: str) -> ResolvedRef | None: + """Return the resolved reference for slot *arg*, or ``None``.""" + return self.slots.get(arg) + + +def resolve_operator_refs(expression: exp.Expression, tables: Tables) -> exp.Expression: + """Attach resolution metadata to every GIQL operator in *expression*. + + Walks the tree's scopes leaves-first, resolves each operator's declared + reference slots against the enclosing scope's visible CTEs plus *tables*, + attaches an :class:`OperatorResolution` under ``node.meta["giql"]``, and + closes with :func:`validate_operator_refs`. + + The pass mutates and returns *expression* in place. It is behavior- + preserving: the attached metadata is additive and the generator ignores it. + + Parameters + ---------- + expression : exp.Expression + The (post-transformer) AST to annotate. + tables : Tables + The registered table configurations. + + Returns + ------- + exp.Expression + The same *expression*, with metadata attached. + """ + seen: set[int] = set() + + for scope in _safe_traverse_scope(expression): + cte_names = frozenset(scope.cte_sources) + for node in scope.walk(): + if isinstance(node, _OPERATORS) and id(node) not in seen: + seen.add(id(node)) + _resolve_operator(node, tables, cte_names) + + # Fallback for any operator a scope walk did not reach (e.g. if scope + # construction failed). Resolving with no visible CTE names keeps the pass + # behavior-preserving: a missed CTE reference simply stays unresolved and + # the generator handles it on its existing path. + for node in expression.walk(): + if isinstance(node, _OPERATORS) and id(node) not in seen: + seen.add(id(node)) + _resolve_operator(node, tables, frozenset()) + + validate_operator_refs(expression) + return expression + + +def _safe_traverse_scope(expression: exp.Expression) -> list[Scope]: + """Return the leaves-first scopes of *expression*, or ``[]`` on failure. + + ``traverse_scope`` raises on a handful of exotic constructs. Swallowing the + failure keeps the pass behavior-preserving: the fallback walk in + :func:`resolve_operator_refs` still annotates every operator, just without + scoped CTE visibility. + """ + try: + return traverse_scope(expression) + except Exception: + return [] + + +def _resolve_operator( + node: exp.Expression, tables: Tables, cte_names: frozenset[str] +) -> None: + """Resolve *node*'s reference slots and attach an :class:`OperatorResolution`.""" + slots: dict[str, ResolvedRef] = {} + + if isinstance(node, GIQLDisjoin): + target_ref = _resolve_target(node.this, tables) + if target_ref is not None: + slots["this"] = target_ref + reference = node.args.get("reference") + ref = _resolve_disjoin_reference(reference, target_ref, cte_names, tables) + if ref is not None: + slots["reference"] = ref + elif isinstance(node, GIQLNearest): + target_ref = _resolve_target(node.this, tables) + if target_ref is not None: + slots["this"] = target_ref + # DISTANCE and the spatial predicates declare only column / literal slots, + # whose resolution metadata is designed by their port issues; the pass + # attaches an (empty-slot) resolution so every operator carries metadata. + + node.meta[META_KEY] = OperatorResolution(type(node).__name__, slots) + + +def _target_name(target: exp.Expression) -> str: + """Extract the target table name from an operator's ``this`` slot. + + Mirrors ``BaseGIQLGenerator._resolve_target_table`` exactly. + """ + if isinstance(target, exp.Table): + return target.name + if isinstance(target, exp.Column): + return target.table if target.table else str(target.this) + return str(target) + + +def _resolve_target(target: exp.Expression, tables: Tables) -> ResolvedRef | None: + """Resolve a target (``this``) slot to a registered-table :class:`ResolvedRef`. + + Returns ``None`` when the target is not a registered table, in which case the + generator raises its existing "not found in tables" error. + """ + name = _target_name(target) + table = tables.get(name) + if table is None: + return None + return ResolvedRef( + kind="registered_table", + name=name, + cols=(table.chrom_col, table.start_col, table.end_col), + table=table, + coverage_skippable=False, + ) + + +def _resolve_disjoin_reference( + reference: exp.Expression | None, + target_ref: ResolvedRef, + cte_names: frozenset[str], + tables: Tables, +) -> ResolvedRef | None: + """Resolve a DISJOIN ``reference`` slot. + + Mirrors ``BaseGIQLGenerator._resolve_disjoin_reference`` exactly, with one + deliberate difference: CTE visibility comes from the operator's scope + (``scope.cte_sources``) rather than the hand-rolled ``_enclosing_cte_names`` + ancestor walk, as epic #114 specifies. The two agree on every supported + shape; any divergence on exotic constructs is invisible in step 1 because + the generator ignores this metadata. + + Returns ``None`` for the unresolvable shapes (reserved prefix, unknown + name, unsupported node type); the generator raises its existing error. + """ + # No reference: default to the target set — a self-reference whose coverage + # EXISTS is provably always true. + if reference is None: + return replace(target_ref, coverage_skippable=True) + + # Subquery reference: a distinct relation assumed to expose canonical + # default columns. Proving equivalence with the target is out of scope, so + # it is never a self-reference. + if isinstance(reference, (exp.Subquery, exp.Select, exp.Union)): + return ResolvedRef( + kind="subquery", + name=None, + cols=_DEFAULT_COLS, + table=None, + coverage_skippable=False, + ) + + # Anything that is not a bare table/CTE name is unsupported. + if not isinstance(reference, (exp.Table, exp.Column, exp.Identifier)): + return None + + ref_name = reference.name + + # The __giql_dj_ prefix names the operator's internal CTEs. + if ref_name.startswith("__giql_dj_"): + return None + + # A CTE from an enclosing WITH shadows a registered table of the same name + # and is assumed to expose canonical default columns. A CTE may hold rows + # distinct from a same-named registered table, so it is never a + # self-reference. + if ref_name in cte_names: + return ResolvedRef( + kind="cte", + name=ref_name, + cols=_DEFAULT_COLS, + table=None, + coverage_skippable=False, + ) + + ref_table = tables.get(ref_name) + if ref_table is not None: + is_self = ref_name == target_ref.name and ref_table is target_ref.table + return ResolvedRef( + kind="registered_table", + name=ref_name, + cols=(ref_table.chrom_col, ref_table.start_col, ref_table.end_col), + table=ref_table, + coverage_skippable=is_self, + ) + + return None + + +def validate_operator_refs(expression: exp.Expression) -> None: + """Assert every GIQL operator carries well-formed resolution metadata. + + The closing validation boundary of pass 1, modeled on ``sqlglot``'s + ``validate_qualify_columns`` and Spark's ``CheckAnalysis``: a pure check + over the annotated tree. For each GIQL operator it asserts that the node + carries an :class:`OperatorResolution`, and that every reference slot which + *was* resolved carries a :class:`ResolvedRef` whose ``kind`` is permitted by + the slot's declared :class:`~giql.expressions.SlotSpec`. + + Consistent with the step-1 zero-behavior-change constraint, an unresolved + reference slot is **not** an error here — that diagnostic is deferred to the + generator (and, in later epic steps, promoted to a GIQL-level message). This + function raises only :class:`ResolutionError`, signalling an internal bug in + the resolver rather than an invalid user query. + + Parameters + ---------- + expression : exp.Expression + The annotated AST to validate. + + Raises + ------ + ResolutionError + If an operator is missing its resolution metadata or a resolved slot is + malformed. + """ + for node in expression.walk(): + if not isinstance(node, _OPERATORS): + continue + + resolution = node.meta.get(META_KEY) + if not isinstance(resolution, OperatorResolution): + raise ResolutionError( + f"{type(node).__name__} is missing resolution metadata; " + "the ResolveOperatorRefs pass did not annotate it." + ) + + specs: tuple[SlotSpec, ...] = getattr(node, "GIQL_SLOTS", ()) + for spec in specs: + if not spec.is_ref_slot: + continue + ref = resolution.slots.get(spec.arg) + if ref is None: + # Deferred: unresolved reference slots are handled by the + # generator on its existing path in step 1. + continue + _validate_ref(ref, spec, type(node).__name__) + + +def _validate_ref(ref: object, spec: SlotSpec, operator: str) -> None: + """Assert a single resolved reference is well-formed against its slot spec.""" + if not isinstance(ref, ResolvedRef): + raise ResolutionError( + f"{operator} slot {spec.arg!r} carries {type(ref).__name__}, " + "expected ResolvedRef." + ) + if ref.kind not in spec.accepts: + raise ResolutionError( + f"{operator} slot {spec.arg!r} resolved to kind {ref.kind!r}, " + f"which is not accepted by the slot (accepts {sorted(spec.accepts)})." + ) + if not ( + isinstance(ref.cols, tuple) + and len(ref.cols) == 3 + and all(isinstance(col, str) for col in ref.cols) + ): + raise ResolutionError( + f"{operator} slot {spec.arg!r} has malformed cols {ref.cols!r}; " + "expected a 3-tuple of column names." + ) + if ref.kind == "registered_table" and ref.table is None: + raise ResolutionError( + f"{operator} slot {spec.arg!r} resolved to a registered table but " + "carries no Table config." + ) + if ref.kind in ("cte", "subquery") and ref.table is not None: + raise ResolutionError( + f"{operator} slot {spec.arg!r} resolved to a {ref.kind} but carries " + "a Table config; CTE and subquery references are assumed canonical." + ) diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 98add46..131b4ea 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -13,6 +13,7 @@ from giql.dialect import GIQLDialect from giql.generators import BaseGIQLGenerator +from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables from giql.transformer import ClusterTransformer @@ -171,6 +172,13 @@ def transpile( ast = merge_transformer.transform(ast) ast = cluster_transformer.transform(ast) + # Pass 1 of the normalization pipeline (epic #114): attach resolution + # metadata to every GIQL operator slot ahead of generation. Behavior- + # preserving in step 1 — the generator still uses its existing resolver + # paths and ignores this metadata. + with _reraise_as_value_error("Resolution error"): + ast = resolve_operator_refs(ast, tables_container) + with _reraise_as_value_error("Transpilation error"): sql = generator.generate(ast) From 548ad695e536c103e07dab6b23299ade65b04b65 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 9 Jun 2026 17:29:46 -0400 Subject: [PATCH 034/142] test: Cover resolver pass and slot descriptor contracts Exercise the ResolveOperatorRefs pass over DISJOIN and NEAREST slot resolution, CTE shadowing and outer-WITH visibility, copy survival of attached metadata, deferral of unresolvable slots, and the validation boundary's rejection arms. Pin the declarative GIQL_SLOTS contract on the operator classes and round-trip arbitrary Table configs through resolution with a Hypothesis property test. --- tests/test_expressions.py | 111 +++++++ tests/test_resolver.py | 617 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 728 insertions(+) create mode 100644 tests/test_expressions.py create mode 100644 tests/test_resolver.py diff --git a/tests/test_expressions.py b/tests/test_expressions.py new file mode 100644 index 0000000..73cc1cd --- /dev/null +++ b/tests/test_expressions.py @@ -0,0 +1,111 @@ +"""Unit tests for the GIQL expression-node slot descriptors. + +Tests pin the declarative ``SlotSpec`` contract on the operator expression +classes: which slots each operator declares and which of them are table-shaped +reference slots that the ``ResolveOperatorRefs`` pass resolves. +""" + +from giql.expressions import Contains +from giql.expressions import GIQLDisjoin +from giql.expressions import GIQLDistance +from giql.expressions import GIQLNearest +from giql.expressions import Intersects +from giql.expressions import SlotSpec +from giql.expressions import SpatialSetPredicate +from giql.expressions import Within + +OPERATOR_CLASSES = ( + GIQLDisjoin, + GIQLNearest, + GIQLDistance, + Intersects, + Contains, + Within, + SpatialSetPredicate, +) + + +class TestSlotSpec: + """Tests for the SlotSpec slot descriptor.""" + + def test_is_ref_slot_with_table_trichotomy_shapes(self): + """Test is_ref_slot for a slot accepting only table shapes. + + Given: + A SlotSpec whose accepted shapes are the registered-table / CTE / + subquery trichotomy. + When: + Reading its is_ref_slot property. + Then: + It should be True. + """ + # Arrange + spec = SlotSpec("reference", frozenset({"registered_table", "cte", "subquery"})) + + # Act & assert + assert spec.is_ref_slot is True + + def test_is_ref_slot_with_column_shape(self): + """Test is_ref_slot for a slot accepting a non-table shape. + + Given: + A SlotSpec whose accepted shapes include a column reference. + When: + Reading its is_ref_slot property. + Then: + It should be False. + """ + # Arrange + spec = SlotSpec("expression", frozenset({"literal_range", "column"})) + + # Act & assert + assert spec.is_ref_slot is False + + def test_is_ref_slot_with_empty_accepts(self): + """Test is_ref_slot for a slot accepting no shapes. + + Given: + A SlotSpec with an empty accepted-shapes set. + When: + Reading its is_ref_slot property. + Then: + It should be False. + """ + # Arrange + spec = SlotSpec("this", frozenset()) + + # Act & assert + assert spec.is_ref_slot is False + + def test_giql_slots_declared_on_every_operator_class(self): + """Test the declarative slot contract across the operator classes. + + Given: + The seven GIQL operator expression classes. + When: + Reading each class's GIQL_SLOTS declaration. + Then: + It should declare a tuple of SlotSpecs on every operator, with + exactly DISJOIN's this/reference and NEAREST's this as the + table-shaped reference slots. + """ + # Arrange + expected_ref_slots = { + (GIQLDisjoin, "this"), + (GIQLDisjoin, "reference"), + (GIQLNearest, "this"), + } + + # Act + declared_ref_slots = { + (cls, spec.arg) + for cls in OPERATOR_CLASSES + for spec in cls.GIQL_SLOTS + if spec.is_ref_slot + } + + # Assert + for cls in OPERATOR_CLASSES: + assert isinstance(cls.GIQL_SLOTS, tuple) + assert all(isinstance(spec, SlotSpec) for spec in cls.GIQL_SLOTS) + assert declared_ref_slots == expected_ref_slots diff --git a/tests/test_resolver.py b/tests/test_resolver.py new file mode 100644 index 0000000..faf53dd --- /dev/null +++ b/tests/test_resolver.py @@ -0,0 +1,617 @@ +"""Tests for the ResolveOperatorRefs normalization pass. + +Tests verify that pass 1 attaches well-formed resolution metadata to every GIQL +operator slot, that its resolution of the table-shaped reference slots mirrors +the generator's existing behavior, and that the closing validation boundary +accepts well-formed metadata while rejecting malformed metadata. +""" + +import pytest +from sqlglot import exp +from sqlglot import parse_one + +from giql.dialect import GIQLDialect +from giql.expressions import GIQLDisjoin +from giql.expressions import GIQLDistance +from giql.expressions import GIQLNearest +from giql.expressions import SpatialSetPredicate +from giql.resolver import META_KEY +from giql.resolver import OperatorResolution +from giql.resolver import ResolutionError +from giql.resolver import ResolvedRef +from giql.resolver import resolve_operator_refs +from giql.resolver import validate_operator_refs +from giql.table import Table +from giql.table import Tables + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import given # noqa: E402 +from hypothesis import settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +# The g_ prefix keeps generated identifiers clear of SQL keywords, which would +# otherwise break parsing when interpolated as a table name. +_identifiers = st.from_regex(r"g_[a-z0-9_]{0,8}", fullmatch=True) + + +def _tables(*names: str) -> Tables: + """Build a Tables container registering each name with default columns.""" + container = Tables() + for name in names: + container.register(name, Table(name)) + return container + + +def _disjoin_node(ast: exp.Expression) -> GIQLDisjoin: + """Return the single GIQLDisjoin node reachable from an annotated AST.""" + return next(n for n in ast.walk() if isinstance(n, GIQLDisjoin)) + + +class TestResolveOperatorRefs: + """Tests for the resolve_operator_refs pass.""" + + def test_resolve_operator_refs_attaches_metadata_to_every_operator(self): + """Test that every GIQL operator receives resolution metadata. + + Given: + A query whose WHERE clause carries a spatial predicate. + When: + Running the resolve pass over the parsed AST. + Then: + It should attach an OperatorResolution under the giql meta key. + """ + # Arrange + ast = parse_one( + "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("peaks")) + + # Assert + predicate = next(n for n in ast.walk() if type(n).__name__ == "Intersects") + assert isinstance(predicate.meta.get(META_KEY), OperatorResolution) + + def test_resolve_operator_refs_resolves_disjoin_target(self): + """Test that a DISJOIN target resolves to its registered table. + + Given: + A DISJOIN over a registered table with custom genomic columns. + When: + Running the resolve pass. + Then: + It should attach a registered-table ResolvedRef carrying the + table's physical column names. + """ + # Arrange + tables = Tables() + tables.register( + "features", + Table("features", chrom_col="chr", start_col="lo", end_col="hi"), + ) + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("this") + assert ref == ResolvedRef( + kind="registered_table", + name="features", + cols=("chr", "lo", "hi"), + table=tables.get("features"), + coverage_skippable=False, + ) + + def test_resolve_operator_refs_omitted_reference_is_self_reference(self): + """Test that an omitted DISJOIN reference is a coverage-skippable self-reference. + + Given: + A DISJOIN with no reference argument. + When: + Running the resolve pass. + Then: + It should resolve the reference slot to the target relation with + coverage_skippable set. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + + # Act + resolve_operator_refs(ast, _tables("features")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "registered_table" + assert ref.name == "features" + assert ref.coverage_skippable is True + + def test_resolve_operator_refs_distinct_registered_reference(self): + """Test that a distinct registered DISJOIN reference is not coverage-skippable. + + Given: + A DISJOIN whose reference is a different registered table. + When: + Running the resolve pass. + Then: + It should resolve to a registered-table reference that is not a + self-reference. + """ + # Arrange + ast = parse_one( + "SELECT * FROM DISJOIN(features, reference := other)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features", "other")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "registered_table" + assert ref.name == "other" + assert ref.coverage_skippable is False + + def test_resolve_operator_refs_cte_reference_shadows_table(self): + """Test that an enclosing CTE reference resolves to a CTE. + + Given: + A DISJOIN reference naming a CTE that shadows a registered table of + the same name. + When: + Running the resolve pass. + Then: + It should resolve to a canonical CTE reference carrying no Table + config. + """ + # Arrange + ast = parse_one( + "WITH other AS (SELECT 1) SELECT * FROM DISJOIN(features, reference := other)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features", "other")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "cte" + assert ref.name == "other" + assert ref.table is None + assert ref.cols == ("chrom", "start", "end") + + def test_resolve_operator_refs_subquery_reference(self): + """Test that a subquery DISJOIN reference resolves to a subquery ref. + + Given: + A DISJOIN whose reference is a parenthesized SELECT. + When: + Running the resolve pass. + Then: + It should resolve to an anonymous canonical subquery reference. + """ + # Arrange + ast = parse_one( + "SELECT * FROM DISJOIN(features, reference := (SELECT * FROM other))", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "subquery" + assert ref.name is None + assert ref.table is None + + def test_resolve_operator_refs_unregistered_target_left_unresolved(self): + """Test that an unregistered target leaves the slot unresolved. + + Given: + A DISJOIN whose target table is not registered. + When: + Running the resolve pass. + Then: + It should attach an OperatorResolution with no resolved slots, + deferring the error to the generator. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(missing)", dialect=GIQLDialect) + + # Act + resolve_operator_refs(ast, Tables()) + + # Assert + resolution = _disjoin_node(ast).meta[META_KEY] + assert resolution.slots == {} + + def test_resolve_operator_refs_reserved_prefix_reference_left_unresolved(self): + """Test that a reserved-prefix reference name leaves the slot unresolved. + + Given: + A DISJOIN reference whose name uses the reserved __giql_dj_ prefix. + When: + Running the resolve pass. + Then: + It should leave the reference slot unresolved, deferring the error + to the generator. + """ + # Arrange + ast = parse_one( + "SELECT * FROM DISJOIN(features, reference := __giql_dj_ref)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features")) + + # Assert + resolution = _disjoin_node(ast).meta[META_KEY] + assert resolution.slot("this") is not None + assert resolution.slot("reference") is None + + def test_resolve_operator_refs_resolves_nearest_target(self): + """Test that a NEAREST target resolves to its registered table. + + Given: + A standalone NEAREST over a registered table. + When: + Running the resolve pass. + Then: + It should attach a registered-table ResolvedRef for the target slot. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1-2', k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("genes")) + + # Assert + nearest = next(n for n in ast.walk() if isinstance(n, GIQLNearest)) + ref = nearest.meta[META_KEY].slot("this") + assert ref.kind == "registered_table" + assert ref.name == "genes" + + def test_resolve_operator_refs_returns_same_expression(self): + """Test that the pass annotates and returns the same AST in place. + + Given: + A parsed GIQL query. + When: + Running the resolve pass. + Then: + It should return the identical expression object it was given. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + + # Act + result = resolve_operator_refs(ast, _tables("features")) + + # Assert + assert result is ast + + def test_resolve_operator_refs_outer_cte_visible_in_derived_table(self): + """Test that an enclosing WITH's CTE is visible to a nested DISJOIN. + + Given: + A DISJOIN nested in a derived table whose reference names a CTE + defined by the enclosing outer WITH clause. + When: + Running the resolve pass. + Then: + It should resolve the reference to a CTE. + """ + # Arrange + ast = parse_one( + "WITH mask AS (SELECT 1) " + "SELECT * FROM (SELECT * FROM DISJOIN(features, reference := mask)) AS d", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "cte" + assert ref.name == "mask" + + def test_resolve_operator_refs_metadata_survives_copy(self): + """Test that attached resolution metadata survives a tree copy. + + Given: + An AST annotated by the resolve pass. + When: + Copying the AST with Expression.copy(). + Then: + It should carry an equal OperatorResolution on the copied operator. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + resolve_operator_refs(ast, _tables("features")) + + # Act + copied = ast.copy() + + # Assert + assert _disjoin_node(copied).meta[META_KEY] == _disjoin_node(ast).meta[META_KEY] + + def test_resolve_operator_refs_annotates_distance_and_set_predicate(self): + """Test that non-reference-slot operators still receive metadata. + + Given: + A query using DISTANCE in the projection and a spatial set + predicate in the WHERE clause. + When: + Running the resolve pass. + Then: + It should attach an empty-slot OperatorResolution to both + operators. + """ + # Arrange + ast = parse_one( + "SELECT DISTANCE(a.interval, b.interval) FROM a, b " + "WHERE a.interval INTERSECTS ANY('chr1:1-2', 'chr2:3-4')", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("a", "b")) + + # Assert + distance = next(n for n in ast.walk() if isinstance(n, GIQLDistance)) + predicate = next(n for n in ast.walk() if isinstance(n, SpatialSetPredicate)) + assert distance.meta[META_KEY].slots == {} + assert predicate.meta[META_KEY].slots == {} + + def test_resolve_operator_refs_unregistered_nearest_target_left_unresolved(self): + """Test that an unregistered NEAREST target leaves the slot unresolved. + + Given: + A NEAREST whose target table is not registered. + When: + Running the resolve pass. + Then: + It should attach an OperatorResolution with no resolved slots, + deferring the error to the generator. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(missing, reference := 'chr1:1-2')", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, Tables()) + + # Assert + nearest = next(n for n in ast.walk() if isinstance(n, GIQLNearest)) + assert nearest.meta[META_KEY].slots == {} + + @settings(max_examples=50) + @given(names=st.lists(_identifiers, min_size=4, max_size=4, unique=True)) + def test_resolve_operator_refs_with_arbitrary_table_config(self, names): + """Test resolution against arbitrary table and column names. + + Given: + Any registered Table with generated identifier-safe table and + chrom/start/end column names. + When: + Resolving a DISJOIN over that table. + Then: + The target ref should carry exactly the configured column names + and validation should pass. + """ + # Arrange + table_name, chrom, start, end = names + tables = Tables() + tables.register( + table_name, + Table(table_name, chrom_col=chrom, start_col=start, end_col=end), + ) + ast = parse_one(f"SELECT * FROM DISJOIN({table_name})", dialect=GIQLDialect) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("this") + assert ref.cols == (chrom, start, end) + validate_operator_refs(ast) + + +class TestValidateOperatorRefs: + """Tests for the validate_operator_refs validation boundary.""" + + def test_validate_operator_refs_accepts_resolved_tree(self): + """Test that validation accepts a well-formed annotated tree. + + Given: + An AST already annotated by the resolve pass. + When: + Running validation over it. + Then: + It should not raise. + """ + # Arrange + ast = parse_one( + "SELECT * FROM DISJOIN(features, reference := other)", + dialect=GIQLDialect, + ) + resolve_operator_refs(ast, _tables("features", "other")) + + # Act & assert + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_missing_metadata(self): + """Test that validation rejects an operator with no metadata. + + Given: + A DISJOIN node that was never annotated by the resolve pass. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the missing metadata. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + + # Act & assert + with pytest.raises(ResolutionError, match="missing resolution metadata"): + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_disallowed_kind(self): + """Test that validation rejects a slot resolved to a disallowed kind. + + Given: + A DISJOIN target slot annotated with a subquery ResolvedRef, a kind + the target slot does not accept. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the rejected kind. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + node = _disjoin_node(ast) + node.meta[META_KEY] = OperatorResolution( + "GIQLDisjoin", + { + "this": ResolvedRef( + kind="subquery", + name=None, + cols=("chrom", "start", "end"), + table=None, + coverage_skippable=False, + ) + }, + ) + + # Act & assert + with pytest.raises(ResolutionError, match="not accepted"): + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_registered_table_without_config(self): + """Test that validation rejects a registered-table ref with no Table. + + Given: + A DISJOIN target annotated as a registered table but carrying no + Table config. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the missing config. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + node = _disjoin_node(ast) + node.meta[META_KEY] = OperatorResolution( + "GIQLDisjoin", + { + "this": ResolvedRef( + kind="registered_table", + name="features", + cols=("chrom", "start", "end"), + table=None, + coverage_skippable=False, + ) + }, + ) + + # Act & assert + with pytest.raises(ResolutionError, match="no Table config"): + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_cte_with_table_config(self): + """Test that validation rejects a CTE ref carrying a Table config. + + Given: + A DISJOIN reference annotated as a CTE but carrying a Table + config, contradicting the canonical assumption for CTEs. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the canonical assumption. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + node = _disjoin_node(ast) + table = Table("features") + node.meta[META_KEY] = OperatorResolution( + "GIQLDisjoin", + { + "reference": ResolvedRef( + kind="cte", + name="features", + cols=("chrom", "start", "end"), + table=table, + coverage_skippable=False, + ) + }, + ) + + # Act & assert + with pytest.raises(ResolutionError, match="assumed canonical"): + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_malformed_cols(self): + """Test that validation rejects a ref with malformed cols. + + Given: + A DISJOIN target annotated with a ResolvedRef whose cols is not a + 3-tuple of column names. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the malformed cols. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + node = _disjoin_node(ast) + table = Table("features") + node.meta[META_KEY] = OperatorResolution( + "GIQLDisjoin", + { + "this": ResolvedRef( + kind="registered_table", + name="features", + cols=("chrom", "start"), + table=table, + coverage_skippable=False, + ) + }, + ) + + # Act & assert + with pytest.raises(ResolutionError, match="malformed cols"): + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_non_resolvedref_slot_value(self): + """Test that validation rejects a slot holding a non-ResolvedRef value. + + Given: + A DISJOIN target slot whose resolution metadata holds a plain + string instead of a ResolvedRef. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the expected type. + """ + # Arrange + ast = parse_one("SELECT * FROM DISJOIN(features)", dialect=GIQLDialect) + node = _disjoin_node(ast) + node.meta[META_KEY] = OperatorResolution( + "GIQLDisjoin", + {"this": "features"}, + ) + + # Act & assert + with pytest.raises(ResolutionError, match="expected ResolvedRef"): + validate_operator_refs(ast) From 661364967e46cec12def0107cd5680a5a6f7aee2 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 9 Jun 2026 23:19:05 -0400 Subject: [PATCH 035/142] refactor: Port DISJOIN emitter to resolver pass metadata Read the DISJOIN target and reference relations from the ResolvedRef metadata attached by the ResolveOperatorRefs pass instead of resolving them at emission time. The coverage EXISTS skip now keys off the resolved coverage_skippable flag, and subquery references render from the AST node since the resolver carries no SQL text. Delete the generator's _resolve_disjoin_reference and _enclosing_cte_names helpers; their semantics live in the resolver pass, with CTE visibility drawn from sqlglot scope sources instead of the hand-rolled ancestor walk. _resolve_target_table stays because the NEAREST emitter still uses it. Unresolvable slots raise the historical diagnostics verbatim through two small reclassification helpers, so user-facing errors are unchanged. --- src/giql/generators/base.py | 204 +++++++++++++----------------------- src/giql/resolver.py | 27 ++--- src/giql/transpile.py | 6 +- 3 files changed, 91 insertions(+), 146 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index ccdcf08..24122cd 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -18,6 +18,9 @@ from giql.expressions import Within from giql.range_parser import ParsedRange from giql.range_parser import RangeParser +from giql.resolver import META_KEY +from giql.resolver import OperatorResolution +from giql.resolver import ResolvedRef from giql.table import Table from giql.table import Tables @@ -283,30 +286,14 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: :return: SQL string (a parenthesized WITH-CTE subquery) for the DISJOIN table """ - # Resolve the target table and its genomic columns. - target_name, (target_chrom, target_start, target_end) = ( - self._resolve_target_table(expression) - ) - target_table = self.tables.get(target_name) - - # The __giql_dj_ prefix names the operator's internal CTEs; a target - # table using it would collide with them. - if target_name.startswith("__giql_dj_"): - raise ValueError( - f"DISJOIN target {target_name!r} uses the reserved " - "'__giql_dj_' prefix, which names the operator's internal " - "CTEs. Rename the table." - ) - - # Resolve the reference relation (defaults to the target set). - ref_from, ref_chrom, ref_start, ref_end, ref_table, is_self_reference = ( - self._resolve_disjoin_reference( - expression, - target_name, - (target_chrom, target_start, target_end), - target_table, - ) - ) + # Unpack the resolution metadata attached by ResolveOperatorRefs (pass 1). + target_ref, ref, ref_from = self._disjoin_resolution(expression) + target_name = target_ref.name + target_chrom, target_start, target_end = target_ref.cols + target_table = target_ref.table + ref_chrom, ref_start, ref_end = ref.cols + ref_table = ref.table + is_self_reference = ref.coverage_skippable # Canonical target endpoints, qualified by the __giql_dj_tgt alias. t_chrom = f't."{target_chrom}"' @@ -886,12 +873,16 @@ def _resolve_nearest_reference( ) def _resolve_target_table( - self, expression: GIQLNearest | GIQLDisjoin + self, expression: GIQLNearest ) -> tuple[str, tuple[str, str, str]]: """Resolve the target table name and its genomic column references. + DISJOIN reads its target from the ResolveOperatorRefs metadata instead + (see :meth:`_disjoin_resolution`); this helper remains for NEAREST + until its port (epic #114, step 3). + :param expression: - GIQLNearest or GIQLDisjoin expression node + GIQLNearest expression node :return: Tuple of (table_name, (chromosome_col, start_col, end_col)) :raises ValueError: @@ -919,113 +910,86 @@ def _resolve_target_table( # Get physical column names from table config return table_name, (table.chrom_col, table.start_col, table.end_col) - def _resolve_disjoin_reference( - self, - expression: GIQLDisjoin, - target_name: str, - target_cols: tuple[str, str, str], - target_table: Table | None, - ) -> tuple[str, str, str, str, Table | None, bool]: - """Resolve the reference relation for a DISJOIN query. - - The reference contributes the breakpoints at which target intervals are - cut. When ``reference`` is omitted it defaults to the target set. - - A bare reference name is resolved against the query's CTEs first: a - name defined by an enclosing ``WITH`` clause shadows a registered table - of the same name (matching SQL scoping) and is assumed to expose - canonical 0-based half-open ``chrom`` / ``start`` / ``end`` columns. A - name with no such CTE but a registered table contributes that table's - column names and coordinate system. A name that is neither is an error. + def _disjoin_resolution( + self, expression: GIQLDisjoin + ) -> tuple[ResolvedRef, ResolvedRef, str]: + """Unpack the DISJOIN resolution attached by ResolveOperatorRefs (pass 1). + + Reads the :class:`~giql.resolver.OperatorResolution` from + ``expression.meta`` and returns ``(target_ref, ref, ref_from)`` where + ``ref_from`` is the text following ``FROM`` inside the ``__giql_dj_ref`` + CTE. A subquery reference carries no name, so it is rendered from the + AST node as an aliased derived table; registered tables and CTEs are + selected from by name. + + The resolver pass deliberately leaves unresolvable slots unresolved + (unregistered target; unsupported reference node type; reference name + using the reserved ``__giql_dj_`` prefix or matching neither a + registered table nor an in-query CTE). For those, and for a target name + using the reserved prefix (which the resolver does resolve), this + re-raises the generator's historical diagnostics verbatim. :param expression: GIQLDisjoin expression node - :param target_name: - Resolved target table name (the default reference) - :param target_cols: - ``(chrom, start, end)`` column names of the target table - :param target_table: - Table config of the target (the default reference config) :return: - Tuple of ``(from_clause, chrom_col, start_col, end_col, table, - is_self_reference)`` where ``from_clause`` is the text following - ``FROM`` inside the ``__giql_dj_ref`` CTE. A subquery or CTE - reference is assumed to expose canonical 0-based half-open - ``chrom`` / ``start`` / ``end`` columns, so ``table`` is ``None`` - for those. ``is_self_reference`` is ``True`` only when the - reference provably resolves to the same registered relation as the - target (omitted reference, or a bare name equal to the target name - that is not shadowed by an enclosing CTE and maps to the same - registered table). It is ``False`` in all other cases, including - subquery and CTE references that may textually duplicate the - target. + Tuple of ``(target_ref, ref, ref_from)`` :raises ValueError: - If the reference is not a table name, a CTE, or a subquery, or if - a bare name resolves to neither a registered table nor a CTE. + If the target or reference slot is unresolved, or a resolved name + uses the reserved ``__giql_dj_`` prefix. """ - reference = expression.args.get("reference") - tc, ts, te = target_cols + resolution = expression.meta.get(META_KEY) + target_ref = ( + resolution.slot("this") + if isinstance(resolution, OperatorResolution) + else None + ) - # No reference: default to the target set (single-mode). - if reference is None: - return target_name, tc, ts, te, target_table, True + # An unresolved target means it is not a registered table. + if target_ref is None: + target = expression.this + if isinstance(target, exp.Table): + target_name = target.name + elif isinstance(target, exp.Column): + target_name = target.table if target.table else str(target.this) + else: + target_name = str(target) + raise ValueError( + f"Target table '{target_name}' not found in tables. " + "Register the table before transpiling." + ) - # Subquery reference: inline it as an aliased derived table. The - # subquery may textually duplicate the target, but proving equivalence - # is out of scope, so treat it as a distinct relation. - if isinstance(reference, (exp.Subquery, exp.Select, exp.Union)): - from_clause = f"{self.sql(reference)} AS __giql_dj_rs" - return ( - from_clause, - DEFAULT_CHROM_COL, - DEFAULT_START_COL, - DEFAULT_END_COL, - None, - False, + # The __giql_dj_ prefix names the operator's internal CTEs; a target + # table using it would collide with them. + if target_ref.name.startswith("__giql_dj_"): + raise ValueError( + f"DISJOIN target {target_ref.name!r} uses the reserved " + "'__giql_dj_' prefix, which names the operator's internal " + "CTEs. Rename the table." ) - # Anything that is not a bare table/CTE name is unsupported. + reference = expression.args.get("reference") + ref = resolution.slot("reference") + if ref is not None: + if ref.kind == "subquery": + return target_ref, ref, f"{self.sql(reference)} AS __giql_dj_rs" + return target_ref, ref, ref.name + + # Unresolved reference: re-classify it and raise the matching + # historical diagnostic. An omitted reference always resolves (to the + # target set), so reference is non-None here. if not isinstance(reference, (exp.Table, exp.Column, exp.Identifier)): raise ValueError( "DISJOIN reference must be a table name, a CTE, or a " f"(SELECT ...) subquery; got {type(reference).__name__}: " f"{reference}" ) - ref_name = reference.name - - # The __giql_dj_ prefix names the operator's internal CTEs. if ref_name.startswith("__giql_dj_"): raise ValueError( f"DISJOIN reference {ref_name!r} uses the reserved " "'__giql_dj_' prefix, which names the operator's internal " "CTEs. Rename the reference relation." ) - - # A CTE from an enclosing WITH shadows a registered table of the same - # name; it is assumed to expose canonical default columns. A CTE may - # contain rows distinct from any same-named registered table, so it is - # never treated as a self-reference. - if ref_name in self._enclosing_cte_names(expression): - return ( - ref_name, - DEFAULT_CHROM_COL, - DEFAULT_START_COL, - DEFAULT_END_COL, - None, - False, - ) - - ref_table = self.tables.get(ref_name) - if ref_table: - return ( - ref_name, - ref_table.chrom_col, - ref_table.start_col, - ref_table.end_col, - ref_table, - ref_name == target_name and ref_table is target_table, - ) raise ValueError( f"DISJOIN reference {ref_name!r} is neither a registered table " "nor a CTE defined in this query. Register the table or define " @@ -1046,26 +1010,6 @@ def _extract_bool_param(param_expr: exp.Expression | None) -> bool: else: return str(param_expr).upper() in ("TRUE", "1", "YES") - @staticmethod - def _enclosing_cte_names(expression: exp.Expression) -> set[str]: - """Collect CTE names visible to an expression from enclosing WITH clauses. - - Walks the ancestor chain and gathers the aliases of every ``WITH``-clause - CTE, so a DISJOIN reference can be recognized as an in-query CTE rather - than a registered table. - """ - names: set[str] = set() - node: exp.Expression | None = expression - while node is not None: - if isinstance(node, exp.Select): - with_clause = node.args.get("with_") or node.args.get("with") - if with_clause is not None: - for cte in with_clause.expressions: - if cte.alias: - names.add(cte.alias) - node = node.parent - return names - def _get_column_refs( self, column_ref: str, diff --git a/src/giql/resolver.py b/src/giql/resolver.py index 07b3a91..112c64e 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -22,13 +22,15 @@ asserts every operator slot carries well-formed resolution metadata, mirroring ``sqlglot``'s ``validate_qualify_columns`` and Spark's ``CheckAnalysis``. -Scope note (epic #114, step 1) ------------------------------- -This is *scaffolding*: it lands with **zero behavior change**. The generator -still uses its existing resolver paths and ignores everything attached here. The -resolution semantics computed for the table-shaped reference slots mirror the -generator's existing ``_resolve_target_table`` / ``_resolve_disjoin_reference`` -/ ``_enclosing_cte_names`` behavior exactly. +Scope note (epic #114, steps 1-2) +--------------------------------- +The pass is behavior-preserving. DISJOIN's emitter +(``BaseGIQLGenerator.giqldisjoin_sql``) consumes the attached metadata (step 2); +the remaining operators still use the generator's legacy resolver paths and +ignore everything attached here until their port issues land. The resolution +semantics computed for the table-shaped reference slots mirror the generator's +historical ``_resolve_target_table`` / ``_resolve_disjoin_reference`` / +``_enclosing_cte_names`` behavior exactly (the latter two now live only here). Two consequences of the zero-behavior-change constraint shape the implementation: @@ -302,12 +304,11 @@ def _resolve_disjoin_reference( ) -> ResolvedRef | None: """Resolve a DISJOIN ``reference`` slot. - Mirrors ``BaseGIQLGenerator._resolve_disjoin_reference`` exactly, with one - deliberate difference: CTE visibility comes from the operator's scope - (``scope.cte_sources``) rather than the hand-rolled ``_enclosing_cte_names`` - ancestor walk, as epic #114 specifies. The two agree on every supported - shape; any divergence on exotic constructs is invisible in step 1 because - the generator ignores this metadata. + Mirrors the generator's historical ``_resolve_disjoin_reference`` exactly + (ported here in epic #114, step 2), with one deliberate difference: CTE + visibility comes from the operator's scope (``scope.cte_sources``) rather + than the hand-rolled ``_enclosing_cte_names`` ancestor walk, as epic #114 + specifies. The two agree on every supported shape. Returns ``None`` for the unresolvable shapes (reserved prefix, unknown name, unsupported node type); the generator raises its existing error. diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 131b4ea..d22c9ca 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -173,9 +173,9 @@ def transpile( ast = cluster_transformer.transform(ast) # Pass 1 of the normalization pipeline (epic #114): attach resolution - # metadata to every GIQL operator slot ahead of generation. Behavior- - # preserving in step 1 — the generator still uses its existing resolver - # paths and ignores this metadata. + # metadata to every GIQL operator slot ahead of generation. DISJOIN's + # emitter consumes this metadata (step 2); the remaining operators still + # use the generator's legacy resolver paths until their ports land. with _reraise_as_value_error("Resolution error"): ast = resolve_operator_refs(ast, tables_container) From d85c41861170f792f3831050bc98a58aaf0c6904 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 9 Jun 2026 23:19:05 -0400 Subject: [PATCH 036/142] test: Pin DISJOIN coverage skip and CTE scope resolution semantics Parametrize the coverage EXISTS skip-vs-keep decision across the five reference shapes: omitted, same-name registered table, distinct table, in-query CTE, and subquery. Pin the CTE visibility semantics the resolver pass inherited from the deleted ancestor walk: WITH RECURSIVE references, set-operation-attached WITH clauses, redeclared inner CTEs, and the invisibility of CTEs declared in sibling derived tables. --- tests/test_disjoin_transpilation.py | 163 ++++++++++++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index b9f3e6e..16c2bc3 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -505,6 +505,63 @@ def test_giqldisjoin_sql_should_emit_one_exists_clause_when_query_mixes_self_and # Assert assert sql.count("EXISTS (") == 1 + @pytest.mark.parametrize( + ("query", "tables", "skips_exists"), + [ + ("SELECT * FROM DISJOIN(features)", ["features"], True), + ( + "SELECT * FROM DISJOIN(features, reference := features)", + ["features"], + True, + ), + ( + "SELECT * FROM DISJOIN(features, reference := refs)", + ["features", "refs"], + False, + ), + ( + "WITH refs AS (SELECT 1) " + "SELECT * FROM DISJOIN(features, reference := refs)", + ["features"], + False, + ), + ( + "SELECT * FROM DISJOIN(features, reference := (SELECT * FROM refs))", + ["features", "refs"], + False, + ), + ], + ids=[ + "omitted-reference", + "same-name-registered-table", + "distinct-registered-table", + "in-query-cte", + "subquery", + ], + ) + def test_giqldisjoin_sql_should_pin_coverage_skippable_across_reference_shapes( + self, query, tables, skips_exists + ): + """Test that the coverage EXISTS skip-vs-keep is preserved per shape. + + Given: + A DISJOIN call across each reference shape — omitted, same-name + registered table, distinct registered table, in-query CTE, and + subquery — whose ResolvedRef.coverage_skippable drives the EXISTS + decision. + When: + Transpiling to SQL. + Then: + It should omit the coverage EXISTS subquery exactly for the + provable self-reference shapes and emit it for every shape that may + hold rows distinct from the target. + """ + # Arrange & act + sql = transpile(query, tables=tables) + + # Assert + assert ("EXISTS (" not in sql) is skips_exists + def test_giqldisjoin_sql_should_emit_exists_clause_when_enclosing_cte_shadows_target_from_outer_scope( self, ): @@ -532,3 +589,109 @@ def test_giqldisjoin_sql_should_emit_exists_clause_when_enclosing_cte_shadows_ta # Assert assert sql.count("EXISTS (") == 1 + + def test_giqldisjoin_sql_should_resolve_recursive_cte_reference(self): + """Test that a WITH RECURSIVE CTE reference resolves to the CTE. + + Given: + A DISJOIN reference naming a CTE declared by WITH RECURSIVE, with + a same-named registered table whose 1-based closed encoding would + shift coordinates if the table were (wrongly) selected. + When: + Transpiling to SQL. + Then: + It should resolve the reference to the CTE — selecting from it by + name, keeping the coverage EXISTS, and applying no coordinate + shift. + """ + # Arrange & act + sql = transpile( + "WITH RECURSIVE refs AS (SELECT 1 UNION ALL SELECT 2) " + "SELECT * FROM DISJOIN(features, reference := refs)", + tables=[ + "features", + Table("refs", coordinate_system="1based", interval_type="closed"), + ], + ) + + # Assert + assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql + assert "EXISTS (" in sql + assert '("start" - 1)' not in sql + + def test_giqldisjoin_sql_should_resolve_set_operation_attached_cte_reference(self): + """Test that a WITH attached to a set operation is visible to DISJOIN. + + Given: + A UNION ALL query whose WITH clause hangs on the set-operation + node and whose first branch passes the CTE name as a DISJOIN + reference, with no same-named registered table. + When: + Transpiling to SQL. + Then: + It should resolve the reference to the CTE per SQL scoping — the + scope-based resolver sees set-operation-attached WITH clauses, + matching how the engine resolves the emitted name at execution + time. + """ + # Arrange & act + sql = transpile( + "WITH refs AS (SELECT 1) " + "SELECT * FROM DISJOIN(features, reference := refs) " + "UNION ALL " + "SELECT * FROM DISJOIN(features)", + tables=["features"], + ) + + # Assert + assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql + assert sql.count("EXISTS (") == 1 + + def test_giqldisjoin_sql_should_resolve_redeclared_inner_cte_reference(self): + """Test that an inner WITH redeclaring an outer CTE name still resolves. + + Given: + A nested query where both the outer query and the inner derived + table declare a CTE named ``refs``, and the DISJOIN inside the + inner scope passes ``refs`` as its reference. + When: + Transpiling to SQL. + Then: + It should resolve the reference to the (shadowing) CTE — selecting + from it by name and keeping the coverage EXISTS. + """ + # Arrange & act + sql = transpile( + "WITH refs AS (SELECT 1) " + "SELECT * FROM (" + "WITH refs AS (SELECT 2) " + "SELECT * FROM DISJOIN(features, reference := refs)" + ") AS sub", + tables=["features"], + ) + + # Assert + assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql + assert sql.count("EXISTS (") == 1 + + def test_giqldisjoin_sql_should_not_see_cte_declared_in_sibling_derived_table( + self, + ): + """Test that a CTE declared inside a sibling derived table stays invisible. + + Given: + An outer DISJOIN whose reference names a CTE declared only inside + a sibling derived table, with no registered table of that name. + When: + Transpiling to SQL. + Then: + It should raise the unknown-reference ValueError — inner WITH + clauses are not in scope for the outer query. + """ + # Arrange, act, & assert + with pytest.raises(ValueError, match="neither a registered table"): + transpile( + "SELECT * FROM DISJOIN(features, reference := refs), " + "(WITH refs AS (SELECT 1) SELECT * FROM refs) AS sub", + tables=["features"], + ) From bb14e28d3803257f938696eaafb1e68618fbe392 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 10:07:02 -0400 Subject: [PATCH 037/142] feat: Add CanonicalizeCoordinates pass with wrapper CTE synthesis Introduce the second normalization pass of the AST pipeline. For every opted-in operator reference slot resolving to a registered table with a non-canonical interval encoding, the pass synthesizes a hidden __giql_canon_* WITH-CTE projecting the relation to 0-based half-open form and repoints both the AST slot and its resolution metadata at the canonical CTE. Synthesis follows the sqlglot eliminate_subqueries recipe: taken names are collected across all scopes, aliases come from name_sequence with find_new_name collision fallback, identical wrapper bodies deduplicate to one CTE, and canonical CTEs are prepended to the outermost WITH so they precede any consumer. Operators opt in through a GIQL_CANONICALIZE class attribute. No operator opts in yet, so the pass is a strict no-op and emitted SQL is unchanged; the per-operator migrations flip the flags and the in-emitter canonical arithmetic remains live until then. The de-canonicalization hook for outermost projections is wired but inert for the same reason. --- src/giql/canonicalizer.py | 355 ++++++++++++++++++++++++++++++++++++++ src/giql/transpile.py | 8 + 2 files changed, 363 insertions(+) create mode 100644 src/giql/canonicalizer.py diff --git a/src/giql/canonicalizer.py b/src/giql/canonicalizer.py new file mode 100644 index 0000000..bb0e34f --- /dev/null +++ b/src/giql/canonicalizer.py @@ -0,0 +1,355 @@ +"""Pass 2 of the GIQL normalization pipeline: ``CanonicalizeCoordinates``. + +This module implements the second normalization pass described in epic #114. It +runs after :func:`giql.resolver.resolve_operator_refs` (pass 1) and consumes the +:class:`~giql.resolver.ResolvedRef` metadata that pass attached to every GIQL +operator slot. For every interval-bearing relation reachable from an operator +slot whose ``ResolvedRef`` carries a :class:`~giql.table.Table` config declaring +a *non-canonical* encoding (``coordinate_system`` / ``interval_type`` other than +``0based`` / ``half_open``), the pass synthesizes a hidden ``__giql_canon_*`` +WITH-CTE that projects the relation to canonical 0-based half-open coordinates +(replicating the :mod:`giql.canonical` arithmetic as SQL in the CTE projection) +and rewrites the operator slot — both the AST node and its ``ResolvedRef`` +metadata — to point at the canonical CTE. + +The synthesis follows ``sqlglot``'s ``eliminate_subqueries`` mechanics +(:mod:`sqlglot.optimizer.eliminate_subqueries`): + +* **Collision-safe aliases.** Candidate names come from + :func:`sqlglot.helper.name_sequence` with the ``__giql_canon_`` prefix, checked + against a taken-set collected across every scope's table sources and CTE + aliases; :func:`sqlglot.helper.find_new_name` resolves any residual collision + against a user identifier. +* **Wrapper-body deduplication.** Two operator slots referencing the same source + under the same encoding share a single canonical CTE — one source is wrapped at + most once per query, keyed on the rendered wrapper body. +* **DAG-ordered insertion.** Canonical CTEs depend only on base (registered) + tables, never on other CTEs, so they are prepended to the outermost ``WITH``; + any existing CTE or main-body operator that now references a canonical CTE + therefore sees it defined first. + +Identity-encoded relations pass through **unwrapped**: a relation already in +0-based half-open form, or one with no ``Table`` config (a CTE or subquery +reference, assumed canonical), is left untouched — the readability-vs-volume +tradeoff the epic calls out (only synthesize a wrapper when canonicalization +actually changes columns). + +Gating (epic #114, step 6) +-------------------------- +The pass is gated per operator by a ``GIQL_CANONICALIZE`` class attribute on the +operator's expression class. An operator opts in by setting +``GIQL_CANONICALIZE = True``; absent or ``False`` (the default for every operator +as of this issue) the pass ignores it entirely. The operator port issues — #122 +(DISJOIN) and #123 (NEAREST / DISTANCE / predicates) — flip these flags as each +operator's emitter is moved off in-emitter canonicalization +(:mod:`giql.canonical`) and onto this pass's output. **With every flag off the +pass is a strict no-op and the emitted SQL is byte-identical**, so the existing +suite is the migration oracle. + +De-canonicalization hook +------------------------- +The outermost ``SELECT`` projection receives a de-canonicalization rewrite for +any output column that a migrated operator emitted in canonical form but that +must land in the user's preferred encoding. With no operator migrated in this +issue that rewrite has nothing to act on; :func:`_decanonicalize_outputs` is the +designed-but-inert hook the port issues will fill in. +""" + +from __future__ import annotations + +from dataclasses import replace + +from sqlglot import exp +from sqlglot.helper import find_new_name +from sqlglot.helper import name_sequence +from sqlglot.optimizer.scope import traverse_scope + +from giql.expressions import Contains +from giql.expressions import GIQLDisjoin +from giql.expressions import GIQLDistance +from giql.expressions import GIQLNearest +from giql.expressions import Intersects +from giql.expressions import SlotSpec +from giql.expressions import SpatialSetPredicate +from giql.expressions import Within +from giql.resolver import META_KEY +from giql.resolver import OperatorResolution +from giql.resolver import ResolvedRef +from giql.table import Table + +__all__ = [ + "CANON_PREFIX", + "canonicalize_coordinates", +] + +#: The alias prefix for every synthesized canonical wrapper CTE. Mirrors the +#: ``__giql_dj_`` reserved prefix the DISJOIN emitter already uses; the leading +#: double underscore keeps the namespace clear of user identifiers. +CANON_PREFIX = "__giql_canon_" + +#: The GIQL operator expression classes the pass inspects — the same set pass 1 +#: annotates. Membership alone does not opt an operator in; the per-class +#: ``GIQL_CANONICALIZE`` flag does (see module docstring). +_OPERATORS: tuple[type[exp.Expression], ...] = ( + GIQLDisjoin, + GIQLNearest, + GIQLDistance, + Intersects, + Contains, + Within, + SpatialSetPredicate, +) + + +def canonicalize_coordinates(expression: exp.Expression) -> exp.Expression: + """Synthesize canonical wrapper CTEs for non-canonical operator operands. + + Walks *expression* for opted-in GIQL operators (those whose expression class + sets ``GIQL_CANONICALIZE = True``), and for each reference slot that resolved + to a registered table with a non-canonical encoding, synthesizes a + ``__giql_canon_*`` WITH-CTE projecting that table to canonical 0-based + half-open coordinates and rewrites the slot (AST node + ``ResolvedRef`` + metadata) to point at the canonical CTE. + + The pass mutates and returns *expression* in place. **When no operator opts + in — the state as of issue #121 — it is a strict no-op: no node is touched + and the emitted SQL is byte-identical.** + + Parameters + ---------- + expression : exp.Expression + The pass-1-annotated AST. + + Returns + ------- + exp.Expression + The same *expression*, with canonical wrapper CTEs inserted and migrated + operator slots rewritten (none, while every flag is off). + """ + targets = _collect_targets(expression) + if not targets: + # No opted-in operator carries a non-canonical operand: strict no-op. + return expression + + taken = _collect_taken_names(expression) + next_name = name_sequence(CANON_PREFIX) + body_to_name: dict[str, str] = {} + new_ctes: list[exp.CTE] = [] + + for node, arg, ref in targets: + body = _canonical_projection(ref) + body_sql = body.sql() + name = body_to_name.get(body_sql) + if name is None: + # First time this exact wrapper body is needed: mint a fresh, + # collision-safe alias and emit one CTE for it. + name = _fresh_name(next_name, taken) + taken.add(name) + body_to_name[body_sql] = name + new_ctes.append(_make_cte(name, body)) + # Subsequent slots with an identical body reuse the same CTE (dedup). + _rewrite_slot(node, arg, ref, name) + + _insert_ctes(expression, new_ctes) + _decanonicalize_outputs(expression, targets) + return expression + + +def _collect_targets( + expression: exp.Expression, +) -> list[tuple[exp.Expression, str, ResolvedRef]]: + """Return ``(node, slot_arg, ref)`` triples needing canonicalization. + + A slot qualifies when its operator opts in (``GIQL_CANONICALIZE``), the slot + is a reference slot that resolved to a registered table, and that table's + declared encoding is non-canonical. CTE / subquery references (assumed + canonical) and identity-encoded tables are skipped — they pass through + unwrapped. + """ + targets: list[tuple[exp.Expression, str, ResolvedRef]] = [] + for node in expression.walk(): + if not isinstance(node, _OPERATORS): + continue + if not getattr(node, "GIQL_CANONICALIZE", False): + continue + resolution = node.meta.get(META_KEY) + if not isinstance(resolution, OperatorResolution): + continue + specs: tuple[SlotSpec, ...] = getattr(node, "GIQL_SLOTS", ()) + for spec in specs: + if not spec.is_ref_slot: + continue + ref = resolution.slots.get(spec.arg) + if ref is None: + continue + # Only registered tables carry a declared encoding; CTE / subquery + # references are assumed canonical and pass through unwrapped. + if ref.kind != "registered_table": + continue + if _is_canonical(ref.table): + continue + targets.append((node, spec.arg, ref)) + return targets + + +def _is_canonical(table: Table | None) -> bool: + """Whether *table*'s declared encoding is already canonical 0-based half-open. + + ``None`` (an unregistered relation) is treated as canonical, matching the + :mod:`giql.canonical` convention. + """ + if table is None: + return True + return table.coordinate_system == "0based" and table.interval_type == "half_open" + + +def _collect_taken_names(expression: exp.Expression) -> set[str]: + """Collect every relation name already in use across all scopes. + + Mirrors ``eliminate_subqueries``'s taken-set: every registered-table source + name plus every CTE alias, so a freshly minted ``__giql_canon_*`` alias can + be proven collision-free. A scope-traversal failure falls back to a plain + tree walk so the set is never under-populated. + """ + taken: set[str] = set() + try: + scopes = traverse_scope(expression) + except Exception: + scopes = [] + for scope in scopes: + for source in scope.sources.values(): + if isinstance(source, exp.Table): + taken.add(source.name) + taken.update(scope.cte_sources) + # Belt-and-suspenders: a raw walk catches any name the scope machinery did + # not surface (e.g. when traverse_scope bailed on an exotic construct). + for table in expression.find_all(exp.Table): + taken.add(table.name) + for cte in expression.find_all(exp.CTE): + taken.add(cte.alias) + return taken + + +def _fresh_name(next_name, taken: set[str]) -> str: + """Return a ``__giql_canon_*`` alias not present in *taken*. + + Draws sequential candidates from the ``name_sequence`` generator and, on the + rare collision with a user identifier, defers to ``find_new_name`` for a + suffixed variant — the same two-step ``eliminate_subqueries`` uses. + """ + candidate = next_name() + if candidate in taken: + candidate = find_new_name(taken, candidate) + return candidate + + +def _canonical_projection(ref: ResolvedRef) -> exp.Select: + """Build the ``SELECT`` body that projects *ref*'s table to canonical form. + + The projection exposes the canonical ``chrom`` / ``start`` / ``end`` columns + under their original physical names, with ``start`` / ``end`` rewritten by + the :mod:`giql.canonical` arithmetic for the table's declared encoding. This + is the interval contract every CTE / subquery reference is assumed to satisfy + (canonical 0-based half-open ``chrom`` / ``start`` / ``end``); operator port + issues #122 / #123 may extend it with pass-through columns as their emitters + require. + """ + chrom, start, end = ref.cols + table = ref.table + relation = ref.name + return exp.select( + exp.alias_(exp.column(chrom), chrom), + exp.alias_(_canonical_start_expr(start, table), start), + exp.alias_(_canonical_end_expr(end, table), end), + ).from_(exp.to_table(relation)) + + +def _canonical_start_expr(start: str, table: Table | None) -> exp.Expression: + """Canonical 0-based half-open start expression for a raw start column. + + SQL analog of :func:`giql.canonical.canonical_start`: + + - ``0based``: ``start`` (identity) + - ``1based``: ``start - 1`` + """ + col = exp.column(start) + if table is None or table.coordinate_system == "0based": + return col + return exp.paren(exp.Sub(this=col, expression=exp.Literal.number(1))) + + +def _canonical_end_expr(end: str, table: Table | None) -> exp.Expression: + """Canonical 0-based half-open end expression for a raw end column. + + SQL analog of :func:`giql.canonical.canonical_end`: + + - ``0based`` / ``half_open``: ``end`` (identity) + - ``0based`` / ``closed``: ``end + 1`` + - ``1based`` / ``half_open``: ``end - 1`` + - ``1based`` / ``closed``: ``end`` (identity) + """ + col = exp.column(end) + if table is None: + return col + key = (table.coordinate_system, table.interval_type) + if key == ("0based", "closed"): + return exp.paren(exp.Add(this=col, expression=exp.Literal.number(1))) + if key == ("1based", "half_open"): + return exp.paren(exp.Sub(this=col, expression=exp.Literal.number(1))) + return col + + +def _make_cte(name: str, body: exp.Select) -> exp.CTE: + """Wrap a projection *body* as a named CTE.""" + return exp.CTE( + this=body, + alias=exp.TableAlias(this=exp.to_identifier(name)), + ) + + +def _rewrite_slot(node: exp.Expression, arg: str, ref: ResolvedRef, name: str) -> None: + """Point operator slot *arg* at the canonical CTE *name*. + + Rewrites both the AST slot (to a bare table reference naming the CTE) and the + attached :class:`~giql.resolver.OperatorResolution` (a fresh + :class:`~giql.resolver.ResolvedRef` of kind ``"cte"`` carrying no ``Table`` + config — the CTE is canonical by construction). The physical column names are + preserved because the CTE re-exposes them under their original names. + """ + new_ref = replace(ref, kind="cte", name=name, table=None) + resolution = node.meta[META_KEY] + resolution.slots[arg] = new_ref + node.set(arg, exp.to_table(name)) + + +def _insert_ctes(expression: exp.Expression, new_ctes: list[exp.CTE]) -> None: + """Prepend *new_ctes* to the outermost ``WITH`` in DAG order. + + Canonical CTEs depend only on base tables, so prepending them guarantees any + existing CTE or main-body reference resolves them. An existing ``WITH`` is + extended in place; otherwise a fresh one is attached to the root query. + """ + if not new_ctes: + return + query = expression.expression if isinstance(expression, exp.DDL) else expression + existing = query.args.get("with_") + if existing is not None: + existing.set("expressions", new_ctes + list(existing.expressions)) + else: + query.set("with_", exp.With(expressions=new_ctes)) + + +def _decanonicalize_outputs( + expression: exp.Expression, + targets: list[tuple[exp.Expression, str, ResolvedRef]], +) -> None: + """De-canonicalize migrated operator outputs in the outermost projection. + + Inert hook (epic #114, step 6). The outermost ``SELECT`` projection list + should rewrite any output column a migrated operator emitted in canonical + form back into the user's preferred encoding. No operator is migrated in + issue #121, so there is nothing to rewrite; the operator port issues (#122, + #123) fill this in alongside flipping their ``GIQL_CANONICALIZE`` flags. + """ + # Intentionally empty until an operator opts in (see module docstring). + return None diff --git a/src/giql/transpile.py b/src/giql/transpile.py index d22c9ca..5bfe894 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -11,6 +11,7 @@ from sqlglot import parse_one +from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs @@ -179,6 +180,13 @@ def transpile( with _reraise_as_value_error("Resolution error"): ast = resolve_operator_refs(ast, tables_container) + # Pass 2 of the normalization pipeline (epic #114): synthesize canonical + # __giql_canon_* wrapper CTEs for non-canonical interval operands of + # opted-in operators (GIQL_CANONICALIZE). No operator opts in yet, so this + # is a strict no-op until the per-operator port issues (#122, #123) land. + with _reraise_as_value_error("Canonicalization error"): + ast = canonicalize_coordinates(ast) + with _reraise_as_value_error("Transpilation error"): sql = generator.generate(ast) From 30d4e25bd77876fe8f5234aa5c46dde4188b3e4a Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 10:07:03 -0400 Subject: [PATCH 038/142] test: Cover CanonicalizeCoordinates synthesis and no-op gating Pin the flags-off no-op for canonical and non-canonical tables, the canonical projection arithmetic across all non-identity encodings and custom column names, identity and CTE/subquery pass-through, slot and metadata repointing, wrapper-body deduplication, alias collision avoidance against user-held names, and WITH-clause insertion order. Hypothesis properties assert wrap-iff-non-canonical over the encoding domain and the unconditional no-op while no operator opts in. --- tests/test_canonicalizer.py | 618 ++++++++++++++++++++++++++++++++++++ 1 file changed, 618 insertions(+) create mode 100644 tests/test_canonicalizer.py diff --git a/tests/test_canonicalizer.py b/tests/test_canonicalizer.py new file mode 100644 index 0000000..efffb0f --- /dev/null +++ b/tests/test_canonicalizer.py @@ -0,0 +1,618 @@ +"""Tests for the CanonicalizeCoordinates normalization pass (pass 2). + +These tests pin three guarantees of :mod:`giql.canonicalizer`: + +* With every operator's ``GIQL_CANONICALIZE`` flag off (the state as of issue + #121) the pass is a strict no-op and the emitted SQL is byte-identical for + both canonical and non-canonical tables. +* With an operator opted in, the synthesized ``__giql_canon_*`` wrapper CTE + projects each declared encoding to canonical 0-based half-open with the correct + arithmetic, identity encodings pass through unwrapped, generated aliases avoid + user-name collisions, identical wrapper bodies deduplicate to one CTE, and the + CTEs are inserted in DAG order on the outermost ``WITH``. +""" + +import itertools + +import pytest +from sqlglot import exp +from sqlglot import parse_one + +from giql.canonicalizer import CANON_PREFIX +from giql.canonicalizer import canonicalize_coordinates +from giql.dialect import GIQLDialect +from giql.expressions import GIQLDisjoin +from giql.generators import BaseGIQLGenerator +from giql.resolver import META_KEY +from giql.resolver import resolve_operator_refs +from giql.table import Table +from giql.table import Tables +from giql.transpile import transpile + +hypothesis = pytest.importorskip("hypothesis") +from hypothesis import given # noqa: E402 +from hypothesis import settings # noqa: E402 +from hypothesis import strategies as st # noqa: E402 + +_COORD_SYSTEMS = ("0based", "1based") +_INTERVAL_TYPES = ("half_open", "closed") + + +def _tables(encoding=("0based", "half_open"), names=("variants",)) -> Tables: + """Build a Tables container registering each name with the given encoding.""" + coordinate_system, interval_type = encoding + container = Tables() + for name in names: + container.register( + name, + Table( + name, + coordinate_system=coordinate_system, + interval_type=interval_type, + ), + ) + return container + + +def _prepare(query: str, tables: Tables) -> exp.Expression: + """Parse *query* and run passes 1 and 2 over the resulting AST.""" + ast = parse_one(query, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables) + return canonicalize_coordinates(ast) + + +def _canon_ctes(ast: exp.Expression) -> list[exp.CTE]: + """Return every synthesized canonical CTE in *ast*, in declaration order.""" + with_ = ast.args.get("with_") + if with_ is None: + return [] + return [c for c in with_.expressions if c.alias.startswith(CANON_PREFIX)] + + +def _disjoin(ast: exp.Expression) -> GIQLDisjoin: + """Return the single GIQLDisjoin node reachable from *ast*.""" + return next(n for n in ast.walk() if isinstance(n, GIQLDisjoin)) + + +@pytest.fixture +def disjoin_opted_in(monkeypatch): + """Opt GIQLDisjoin into canonicalization for the duration of a test.""" + monkeypatch.setattr(GIQLDisjoin, "GIQL_CANONICALIZE", True, raising=False) + return GIQLDisjoin + + +class TestNoOpWhenFlagsOff: + """The pass touches nothing while every GIQL_CANONICALIZE flag is off.""" + + def test_canonical_table_sql_unchanged(self): + """Test that a canonical table's SQL is byte-identical through the pass. + + Given: + A DISJOIN over a canonical (0based/half_open) registered table and no + operator opted into canonicalization. + When: + Transpiling the query. + Then: + The pass is inert and the SQL matches a run with the pass bypassed. + """ + # Arrange + query = "SELECT * FROM DISJOIN(variants)" + tables = _tables(("0based", "half_open")) + ast = resolve_operator_refs(parse_one(query, dialect=GIQLDialect), tables) + expected = BaseGIQLGenerator(tables=tables).generate(ast) + + # Act + actual = transpile(query, tables=[Table("variants")]) + + # Assert + assert actual == expected + assert CANON_PREFIX not in actual + + def test_non_canonical_table_sql_unchanged(self): + """Test that a non-canonical table's SQL is byte-identical through the pass. + + Given: + A DISJOIN over a non-canonical (1based/closed) registered table and no + operator opted into canonicalization. + When: + Transpiling the query and comparing against a pass-bypassed run. + Then: + No wrapper CTE is synthesized and the SQL is unchanged. + """ + # Arrange + query = "SELECT * FROM DISJOIN(variants)" + variants = Table("variants", coordinate_system="1based", interval_type="closed") + tables = Tables() + tables.register("variants", variants) + ast = resolve_operator_refs(parse_one(query, dialect=GIQLDialect), tables) + expected = BaseGIQLGenerator(tables=tables).generate(ast) + + # Act + actual = transpile(query, tables=[variants]) + + # Assert + assert actual == expected + assert CANON_PREFIX not in actual + + def test_pass_returns_expression_with_no_with_added(self): + """Test that the no-op pass adds no WITH clause to the AST. + + Given: + A non-canonical DISJOIN AST and no operator opted in. + When: + Running canonicalize_coordinates directly. + Then: + No canonical CTE is present on the returned tree. + """ + # Arrange + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + assert _canon_ctes(ast) == [] + + +class TestProjectionArithmetic: + """The wrapper CTE replicates canonical_start/canonical_end per encoding.""" + + def test_zero_based_closed_projection(self, disjoin_opted_in): + """Test the canonical projection for a 0based/closed table. + + Given: + A DISJOIN over a 0based/closed table with an operator opted in. + When: + Running pass 2. + Then: + The wrapper projects start unchanged and end + 1. + """ + # Arrange + tables = _tables(("0based", "closed")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + body = _canon_ctes(ast)[0].this.sql() + assert "start AS start" in body + assert "(end + 1) AS end" in body + + def test_one_based_half_open_projection(self, disjoin_opted_in): + """Test the canonical projection for a 1based/half_open table. + + Given: + A DISJOIN over a 1based/half_open table with an operator opted in. + When: + Running pass 2. + Then: + The wrapper projects start - 1 and end - 1. + """ + # Arrange + tables = _tables(("1based", "half_open")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + body = _canon_ctes(ast)[0].this.sql() + assert "(start - 1) AS start" in body + assert "(end - 1) AS end" in body + + def test_one_based_closed_projection(self, disjoin_opted_in): + """Test the canonical projection for a 1based/closed table. + + Given: + A DISJOIN over a 1based/closed table with an operator opted in. + When: + Running pass 2. + Then: + The wrapper projects start - 1 and end unchanged. + """ + # Arrange + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + body = _canon_ctes(ast)[0].this.sql() + assert "(start - 1) AS start" in body + assert "end AS end" in body + + def test_projection_preserves_custom_column_names(self, disjoin_opted_in): + """Test the projection uses a table's physical column names. + + Given: + A 1based/closed table with custom chrom/start/end column names. + When: + Running pass 2. + Then: + The wrapper exposes the canonical interval under those same names. + """ + # Arrange + variants = Table( + "variants", + chrom_col="chr", + start_col="lo", + end_col="hi", + coordinate_system="1based", + interval_type="closed", + ) + tables = Tables() + tables.register("variants", variants) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + body = _canon_ctes(ast)[0].this.sql() + assert "chr AS chr" in body + assert "(lo - 1) AS lo" in body + assert "hi AS hi" in body + + +class TestPassThrough: + """Identity-encoded and non-table references pass through unwrapped.""" + + def test_identity_encoding_not_wrapped(self, disjoin_opted_in): + """Test that a canonical table is not wrapped even when opted in. + + Given: + A DISJOIN over a canonical (0based/half_open) table with the operator + opted into canonicalization. + When: + Running pass 2. + Then: + No wrapper CTE is synthesized and the slot metadata is unchanged. + """ + # Arrange + tables = _tables(("0based", "half_open")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + assert _canon_ctes(ast) == [] + ref = _disjoin(ast).meta[META_KEY].slot("this") + assert ref.kind == "registered_table" + + def test_cte_reference_not_wrapped(self, disjoin_opted_in): + """Test that a CTE reference passes through unwrapped. + + Given: + A DISJOIN whose reference resolves to an enclosing CTE while its + target is a non-canonical registered table. + When: + Running pass 2. + Then: + Only the target is wrapped; the CTE reference is left as a CTE ref. + """ + # Arrange + query = ( + "WITH foo AS (SELECT * FROM other) " + "SELECT * FROM DISJOIN(variants, reference := foo)" + ) + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare(query, tables) + + # Assert + slots = _disjoin(ast).meta[META_KEY] + assert slots.slot("this").kind == "cte" # target rewritten to canon CTE + assert slots.slot("this").name.startswith(CANON_PREFIX) + assert slots.slot("reference").name == "foo" # user CTE untouched + + def test_subquery_reference_not_wrapped(self, disjoin_opted_in): + """Test that a subquery reference passes through unwrapped. + + Given: + A DISJOIN whose reference is a subquery and whose target is a + non-canonical registered table. + When: + Running pass 2. + Then: + The subquery reference stays a subquery ref carrying no Table config. + """ + # Arrange + query = "SELECT * FROM DISJOIN(variants, reference := (SELECT * FROM other))" + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare(query, tables) + + # Assert + ref = _disjoin(ast).meta[META_KEY].slot("reference") + assert ref.kind == "subquery" + assert ref.table is None + + +class TestSlotRewrite: + """A wrapped slot is rewritten in both the AST and its metadata.""" + + def test_metadata_rewritten_to_canonical_cte(self, disjoin_opted_in): + """Test that a wrapped slot's ResolvedRef points at the canonical CTE. + + Given: + A DISJOIN over a non-canonical table with the operator opted in. + When: + Running pass 2. + Then: + The slot's ResolvedRef becomes a CTE ref naming the canon CTE and + carrying no Table config. + """ + # Arrange + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + ref = _disjoin(ast).meta[META_KEY].slot("this") + assert ref.kind == "cte" + assert ref.name.startswith(CANON_PREFIX) + assert ref.table is None + assert ref.cols == ("chrom", "start", "end") + + def test_ast_slot_points_at_canonical_cte(self, disjoin_opted_in): + """Test that a wrapped slot's AST node names the canonical CTE. + + Given: + A DISJOIN over a non-canonical table with the operator opted in. + When: + Running pass 2. + Then: + The operator's target AST node references the canon CTE by name. + """ + # Arrange + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + target = _disjoin(ast).this + assert isinstance(target, exp.Table) + assert target.name.startswith(CANON_PREFIX) + + +class TestDeduplication: + """Identical wrapper bodies collapse to a single canonical CTE.""" + + def test_same_table_two_slots_dedup_to_one_cte(self, disjoin_opted_in): + """Test that two slots over the same table share one canonical CTE. + + Given: + A self-referencing DISJOIN over a non-canonical table (both target + and reference resolve to the same table) with the operator opted in. + When: + Running pass 2. + Then: + Exactly one canonical CTE is synthesized and both slots point at it. + """ + # Arrange + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + ctes = _canon_ctes(ast) + assert len(ctes) == 1 + slots = _disjoin(ast).meta[META_KEY] + assert slots.slot("this").name == slots.slot("reference").name == ctes[0].alias + + def test_distinct_tables_get_distinct_ctes(self, disjoin_opted_in): + """Test that two distinct non-canonical tables get distinct CTEs. + + Given: + A DISJOIN whose target and reference are two distinct non-canonical + registered tables, with the operator opted in. + When: + Running pass 2. + Then: + Two distinct canonical CTEs are synthesized. + """ + # Arrange + tables = _tables(("1based", "closed"), names=("variants", "genes")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants, reference := genes)", tables) + + # Assert + ctes = _canon_ctes(ast) + assert len(ctes) == 2 + slots = _disjoin(ast).meta[META_KEY] + assert slots.slot("this").name != slots.slot("reference").name + + +class TestAliasCollision: + """Generated aliases avoid colliding with user identifiers.""" + + def test_alias_avoids_user_cte_collision(self, disjoin_opted_in): + """Test that a generated alias dodges a same-named user CTE. + + Given: + A query containing a user CTE literally named __giql_canon_0 and a + non-canonical DISJOIN, with the operator opted in. + When: + Running pass 2. + Then: + The synthesized CTE takes a distinct, non-colliding alias. + """ + # Arrange + query = ( + f"WITH {CANON_PREFIX}0 AS (SELECT * FROM other) " + "SELECT * FROM DISJOIN(variants)" + ) + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare(query, tables) + + # Assert + synthesized = _disjoin(ast).meta[META_KEY].slot("this").name + assert synthesized.startswith(CANON_PREFIX) + assert synthesized != f"{CANON_PREFIX}0" + + def test_alias_avoids_user_table_collision(self, disjoin_opted_in): + """Test that a generated alias dodges a same-named user table. + + Given: + A query joining a user table literally named __giql_canon_0 alongside + a non-canonical DISJOIN, with the operator opted in. + When: + Running pass 2. + Then: + The synthesized CTE alias does not collide with that table name. + """ + # Arrange + query = ( + f"SELECT * FROM DISJOIN(variants) AS d JOIN {CANON_PREFIX}0 AS u ON 1 = 1" + ) + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare(query, tables) + + # Assert + synthesized = _disjoin(ast).meta[META_KEY].slot("this").name + assert synthesized != f"{CANON_PREFIX}0" + + +class TestDagOrdering: + """Canonical CTEs are inserted before the user CTEs that may reference them.""" + + def test_canon_cte_prepended_before_existing_cte(self, disjoin_opted_in): + """Test that a synthesized CTE precedes existing user CTEs in the WITH. + + Given: + A query with a user CTE and a non-canonical DISJOIN, opted in. + When: + Running pass 2. + Then: + The canonical CTE is the first entry in the outermost WITH clause. + """ + # Arrange + query = "WITH userfoo AS (SELECT * FROM other) SELECT * FROM DISJOIN(variants)" + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare(query, tables) + + # Assert + with_exprs = ast.args["with_"].expressions + assert with_exprs[0].alias.startswith(CANON_PREFIX) + assert "userfoo" in {c.alias for c in with_exprs} + + def test_canon_cte_created_when_no_existing_with(self, disjoin_opted_in): + """Test that a fresh WITH clause is created when none exists. + + Given: + A non-canonical DISJOIN with no enclosing WITH, opted in. + When: + Running pass 2. + Then: + A WITH clause is attached carrying the canonical CTE. + """ + # Arrange + tables = _tables(("1based", "closed")) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + assert ast.args.get("with_") is not None + assert len(_canon_ctes(ast)) == 1 + + +class TestEncodingInvariants: + """Property-based coverage over the full encoding domain.""" + + @settings(max_examples=25, deadline=None) + @given( + coordinate_system=st.sampled_from(_COORD_SYSTEMS), + interval_type=st.sampled_from(_INTERVAL_TYPES), + ) + def test_wrap_iff_non_canonical(self, coordinate_system, interval_type): + """Test that a table is wrapped exactly when its encoding is non-canonical. + + Given: + A DISJOIN over a registered table with any encoding combination, with + the operator opted into canonicalization. + When: + Running passes 1 and 2. + Then: + A canonical CTE exists iff the encoding is not 0based/half_open, and a + wrapped slot always becomes a Table-free CTE ref. + """ + # Arrange + # A plain try/finally toggles the class flag: @given re-runs the body + # many times under one function-scoped fixture, so monkeypatch is unsafe. + GIQLDisjoin.GIQL_CANONICALIZE = True + tables = _tables((coordinate_system, interval_type)) + is_canonical = coordinate_system == "0based" and interval_type == "half_open" + + # Act + try: + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + finally: + del GIQLDisjoin.GIQL_CANONICALIZE + + # Assert + ctes = _canon_ctes(ast) + ref = _disjoin(ast).meta[META_KEY].slot("this") + if is_canonical: + assert ctes == [] + assert ref.kind == "registered_table" + else: + assert len(ctes) == 1 + assert ref.kind == "cte" + assert ref.table is None + + @settings(max_examples=25, deadline=None) + @given( + coordinate_system=st.sampled_from(_COORD_SYSTEMS), + interval_type=st.sampled_from(_INTERVAL_TYPES), + ) + def test_flags_off_is_always_noop(self, coordinate_system, interval_type): + """Test that the pass never mutates the tree while flags are off. + + Given: + A DISJOIN over a registered table with any encoding and no operator + opted in. + When: + Running passes 1 and 2. + Then: + No canonical CTE is ever synthesized. + """ + # Arrange + tables = _tables((coordinate_system, interval_type)) + + # Act + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Assert + assert _canon_ctes(ast) == [] + + +def test_all_encoding_pairs_covered(): + """Test that the sampled encoding domain spans every coordinate/type pair. + + Given: + The coordinate-system and interval-type domains under test. + When: + Enumerating their product. + Then: + Exactly four encoding combinations exist, one of them canonical. + """ + # Arrange + pairs = list(itertools.product(_COORD_SYSTEMS, _INTERVAL_TYPES)) + + # Act + canonical = [p for p in pairs if p == ("0based", "half_open")] + + # Assert + assert len(pairs) == 4 + assert len(canonical) == 1 From 4a1ca86864f36c7d6f16b84b4ffedbc1e2973b33 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 10:25:30 -0400 Subject: [PATCH 039/142] refactor: Port NEAREST emitter to resolver pass metadata Resolve NEAREST's reference slot in the ResolveOperatorRefs pass and read both target and reference from the attached metadata at emission time. A new ResolvedInterval dataclass models the non-table reference forms: a literal genomic range carrying pre-canonicalized endpoints, a column reference carrying raw alias-qualified column SQL, and the implicit LATERAL outer form detected from scope ancestry. A companion SlotDeferral channel records why a slot could not be resolved so the emitter raises the historical diagnostics verbatim. Delete the generator's _resolve_target_table, _resolve_nearest_reference, and _find_outer_table_in_lateral_join helpers; their semantics live in the resolver, including the alias-to-table mapping previously built by select_sql state. Coordinate canonicalization stays in the emitter, wrapping the raw endpoint SQL with the Table config the metadata carries, until the CanonicalizeCoordinates migration retires it. The emitter lazily resolves when metadata is absent so direct generate() callers behave identically to the transpile() pipeline. --- src/giql/generators/base.py | 296 ++++++++---------------- src/giql/resolver.py | 446 +++++++++++++++++++++++++++++++++--- 2 files changed, 516 insertions(+), 226 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 24122cd..631d8dc 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -20,7 +20,9 @@ from giql.range_parser import RangeParser from giql.resolver import META_KEY from giql.resolver import OperatorResolution +from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef +from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables @@ -135,13 +137,42 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: # Detect mode mode = self._detect_nearest_mode(expression) - # Resolve target table - table_name, (target_chrom, target_start, target_end) = ( - self._resolve_target_table(expression) - ) + # Unpack the resolution metadata attached by ResolveOperatorRefs (pass 1). + resolution = self._nearest_resolution(expression) + + # Target (already a registered-table ResolvedRef from the pass). An + # unresolved target means it is not a registered table; raise the + # historical diagnostic. + target_ref = resolution.slot("this") if resolution is not None else None + if not isinstance(target_ref, ResolvedRef): + target = expression.this + if isinstance(target, exp.Table): + target_name = target.name + elif isinstance(target, exp.Column): + target_name = target.table if target.table else str(target.this) + else: + target_name = str(target) + raise ValueError( + f"Target table '{target_name}' not found in tables. " + "Register the table before transpiling." + ) + table_name = target_ref.name + target_chrom, target_start, target_end = target_ref.cols + target_table = target_ref.table - # Resolve reference - ref_chrom, ref_start, ref_end = self._resolve_nearest_reference(expression, mode) + # Reference interval (a ResolvedInterval from the pass). An unresolved + # reference re-raises the generator's historical diagnostic. + ref = resolution.slot("reference") + if not isinstance(ref, ResolvedInterval): + self._raise_nearest_reference_error(expression, mode, resolution) + if ref.kind == "literal_range": + # Literal endpoints are already canonical 0-based half-open. + ref_chrom, ref_start, ref_end = ref.chrom, ref.start, ref.end + else: + # Column / implicit-outer endpoints are raw; canonicalize here. + ref_chrom = ref.chrom + ref_start = canonical_start(ref.start, ref.table) + ref_end = canonical_end(ref.end, ref.table) # Extract parameters k = expression.args.get("k") @@ -153,43 +184,14 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: is_stranded = self._extract_bool_param(expression.args.get("stranded")) is_signed = self._extract_bool_param(expression.args.get("signed")) - target_table = self.tables.get(table_name) if self.tables else None - - # Resolve strand columns if stranded mode + # Resolve strand columns if stranded mode. The reference strand is + # carried on the resolved interval (a literal's strand, an explicit + # column's strand, or the outer table's strand for an implicit + # reference — already gated to preserve the historical divergence). ref_strand = None target_strand = None if is_stranded: - # Get strand column for reference - reference = expression.args.get("reference") - if reference: - reference_sql = self.sql(reference) - - # Check if reference is a literal string or column reference - if reference_sql.startswith("'") or reference_sql.startswith('"'): - # Literal reference - parse for strand - range_str = reference_sql.strip("'\"") - from giql.range_parser import RangeParser - - parsed_range = RangeParser.parse(range_str).to_zero_based_half_open() - if parsed_range.strand: - ref_strand = f"'{parsed_range.strand}'" - else: - # Column reference - get strand column - ref_cols = self._get_column_refs( - reference_sql, None, include_strand=True - ) - if len(ref_cols) == 4: - ref_strand = ref_cols[3] - else: - # Implicit reference in correlated mode - get strand from outer table - outer_table = self._find_outer_table_in_lateral_join(expression) - if outer_table and self.tables: - actual_table = self._alias_to_table.get(outer_table, outer_table) - table = self.tables.get(actual_table) - if table and table.strand_col: - ref_strand = f'{outer_table}."{table.strand_col}"' - - # Get strand column for target table + ref_strand = ref.strand if target_table and target_table.strand_col: target_strand = f'{table_name}."{target_table.strand_col}"' @@ -729,186 +731,86 @@ def _detect_nearest_mode( # (validation will catch missing reference errors later) return "correlated" - def _find_outer_table_in_lateral_join(self, expression: GIQLNearest) -> str | None: - """Find the outer table name in a LATERAL join context. + def _nearest_resolution(self, expression: GIQLNearest) -> OperatorResolution | None: + """Return the NEAREST resolution attached by ResolveOperatorRefs (pass 1). - Walks up the AST to find the JOIN clause and extracts the outer table - that the LATERAL subquery is correlated with. + The transpile pipeline attaches an + :class:`~giql.resolver.OperatorResolution` before generation, and it + survives the generator's defensive tree copy. Direct callers that invoke + ``generate`` / ``giqlnearest_sql`` without running the pass (notably unit + tests) get the metadata resolved lazily here against this generator's + registered tables, so the emit path always reads resolved metadata. :param expression: GIQLNearest expression node :return: - Table name or alias of the outer table, or None if not found + The attached :class:`~giql.resolver.OperatorResolution`, or ``None`` + if resolution did not produce one. """ - # Walk up the AST to find the JOIN - current = expression - while current: - parent = current.parent - if not parent: - break - - # Check if parent is a Lateral expression - if isinstance(parent, exp.Lateral): - # Continue up to find the Join - current = parent - continue - - # Check if parent is a Join - if isinstance(parent, exp.Join): - # The outer table is in the parent Select's FROM clause - # or in previous joins - select = parent.parent - if isinstance(select, exp.Select): - # Get the FROM clause - from_expr = select.args.get("from_") - if from_expr: - # Extract table from FROM - table_expr = from_expr.this - if isinstance(table_expr, exp.Table): - # Return alias if it exists, otherwise table name - return table_expr.alias or table_expr.name - elif isinstance(table_expr, exp.Alias): - return table_expr.alias - break - - current = parent - - return None + resolution = expression.meta.get(META_KEY) + if not isinstance(resolution, OperatorResolution): + resolve_operator_refs(expression.root(), self.tables) + resolution = expression.meta.get(META_KEY) + return resolution if isinstance(resolution, OperatorResolution) else None - def _resolve_nearest_reference( - self, expression: GIQLNearest, mode: str - ) -> tuple[str, str, str] | tuple[str, str, str, str]: - """Resolve the reference position for NEAREST queries. + def _raise_nearest_reference_error( + self, + expression: GIQLNearest, + mode: str, + resolution: OperatorResolution | None, + ) -> None: + """Raise the historical diagnostic for an unresolved NEAREST reference. + + ResolveOperatorRefs (pass 1) defers a reference slot it cannot resolve; + this re-raises the generator's pre-pass error verbatim. The implicit- + outer failures rely on the :class:`~giql.resolver.SlotDeferral` the pass + records (the ancestor walk that distinguished them now lives in the + pass); a literal-range parse failure is reproduced by re-parsing. :param expression: GIQLNearest expression node :param mode: "standalone" or "correlated" - :return: - Tuple of (chromosome, start, end) or (chromosome, start, end, strand) - Returns SQL expressions (column refs for correlated, literals for standalone) - Endpoints are canonicalized to 0-based half-open: literal references via - :meth:`~giql.range_parser.RangeParser.to_zero_based_half_open`, column - references via :func:`giql.canonical.canonical_start` / - :func:`giql.canonical.canonical_end`. + :param resolution: + The attached resolution metadata, if any :raises ValueError: - If reference is missing in standalone mode or invalid format + Always — with the matching historical message """ reference = expression.args.get("reference") - if mode == "standalone": - if not reference: + if reference is None: + # Implicit-outer reference: standalone mode (unreachable in practice, + # since an absent reference is classified as correlated) keeps the + # historical message; otherwise consult the recorded deferral. + if mode == "standalone": raise ValueError( "NEAREST in standalone mode requires explicit reference parameter" ) - - # Get SQL representation of reference - reference_sql = self.sql(reference) - - # Check if it's a literal range string - if reference_sql.startswith("'") or reference_sql.startswith('"'): - # Parse literal genomic range - range_str = reference_sql.strip("'\"") - try: - parsed_range = RangeParser.parse(range_str).to_zero_based_half_open() - # Return as SQL literals - return ( - f"'{parsed_range.chromosome}'", - str(parsed_range.start), - str(parsed_range.end), - ) - except Exception as e: - raise ValueError( - f"Could not parse reference genomic range: " - f"{range_str}. Error: {e}" - ) - else: - # Column reference - resolve and canonicalize - chrom, start, end = self._get_column_refs(reference_sql, None) - ref_table = self._resolve_table(reference_sql) - return ( - chrom, - canonical_start(start, ref_table), - canonical_end(end, ref_table), - ) - - else: # correlated mode - if reference: - # Explicit reference in correlated mode (e.g., peaks.interval) - reference_sql = self.sql(reference) - chrom, start, end = self._get_column_refs(reference_sql, None) - ref_table = self._resolve_table(reference_sql) - return ( - chrom, - canonical_start(start, ref_table), - canonical_end(end, ref_table), - ) - else: - # Implicit reference - resolve from outer table in LATERAL join - outer_table = self._find_outer_table_in_lateral_join(expression) - if not outer_table: - raise ValueError( - "Could not find outer table in LATERAL join context. " - "Please specify reference parameter explicitly." - ) - - # Look up the table to find the genomic column - # Check if outer_table is an alias - actual_table = self._alias_to_table.get(outer_table, outer_table) - table = self.tables.get(actual_table) - - if not table: - raise ValueError( - f"Outer table '{outer_table}' not found in tables. " - "Please specify reference parameter explicitly." - ) - - # Build column references using the outer table and genomic column - reference_sql = f"{outer_table}.{table.genomic_col}" - chrom, start, end = self._get_column_refs(reference_sql, None) - return ( - chrom, - canonical_start(start, table), - canonical_end(end, table), + deferral = ( + resolution.deferral("reference") if resolution is not None else None + ) + if deferral is not None and deferral.reason == "implicit_outer_unregistered": + raise ValueError( + f"Outer table '{deferral.detail}' not found in tables. " + "Please specify reference parameter explicitly." ) - - def _resolve_target_table( - self, expression: GIQLNearest - ) -> tuple[str, tuple[str, str, str]]: - """Resolve the target table name and its genomic column references. - - DISJOIN reads its target from the ResolveOperatorRefs metadata instead - (see :meth:`_disjoin_resolution`); this helper remains for NEAREST - until its port (epic #114, step 3). - - :param expression: - GIQLNearest expression node - :return: - Tuple of (table_name, (chromosome_col, start_col, end_col)) - :raises ValueError: - If target table is not found or doesn't have genomic columns - """ - # Extract target table from 'this' argument - target = expression.this - - if isinstance(target, exp.Table): - table_name = target.name - elif isinstance(target, exp.Column): - # If it's a column reference, extract table name - table_name = target.table if target.table else str(target.this) - else: - # Try to extract as string - table_name = str(target) - - table = self.tables.get(table_name) - if not table: raise ValueError( - f"Target table '{table_name}' not found in tables. " - "Register the table before transpiling." + "Could not find outer table in LATERAL join context. " + "Please specify reference parameter explicitly." ) - # Get physical column names from table config - return table_name, (table.chrom_col, table.start_col, table.end_col) + # An explicit reference that deferred is a literal range that failed to + # parse (column references always resolve). Re-parse to surface the + # original parse error in the historical message. + reference_sql = self.sql(reference) + range_str = reference_sql.strip("'\"") + try: + RangeParser.parse(range_str).to_zero_based_half_open() + except Exception as e: + raise ValueError( + f"Could not parse reference genomic range: {range_str}. Error: {e}" + ) + raise ValueError(f"Could not parse reference genomic range: {range_str}.") def _disjoin_resolution( self, expression: GIQLDisjoin diff --git a/src/giql/resolver.py b/src/giql/resolver.py index 112c64e..c2fac29 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -22,15 +22,18 @@ asserts every operator slot carries well-formed resolution metadata, mirroring ``sqlglot``'s ``validate_qualify_columns`` and Spark's ``CheckAnalysis``. -Scope note (epic #114, steps 1-2) +Scope note (epic #114, steps 1-3) --------------------------------- The pass is behavior-preserving. DISJOIN's emitter -(``BaseGIQLGenerator.giqldisjoin_sql``) consumes the attached metadata (step 2); -the remaining operators still use the generator's legacy resolver paths and -ignore everything attached here until their port issues land. The resolution -semantics computed for the table-shaped reference slots mirror the generator's -historical ``_resolve_target_table`` / ``_resolve_disjoin_reference`` / -``_enclosing_cte_names`` behavior exactly (the latter two now live only here). +(``BaseGIQLGenerator.giqldisjoin_sql``, step 2) and NEAREST's emitter +(``BaseGIQLGenerator.giqlnearest_sql``, step 3) consume the attached metadata; +DISTANCE and the spatial predicates still use the generator's legacy resolver +paths and ignore everything attached here until their port issues land. The +resolution semantics computed here mirror the generator's historical +``_resolve_target_table`` / ``_resolve_disjoin_reference`` / +``_enclosing_cte_names`` (DISJOIN) and ``_resolve_nearest_reference`` / +``_find_outer_table_in_lateral_join`` (NEAREST) behavior exactly; all of those +helpers now live only here. Two consequences of the zero-behavior-change constraint shape the implementation: @@ -40,17 +43,19 @@ prefix, unsupported reference shape) the pass simply leaves that slot unresolved and the generator raises its existing error exactly as before. Promoting these into GIQL-level diagnostics is a later epic step. -* Only *reference slots* — slots whose accepted shapes are the table - trichotomy (DISJOIN ``this``/``reference`` and NEAREST ``this``) — are - resolved to a :class:`ResolvedRef` here. The column / literal / - implicit-outer slots of NEAREST, DISTANCE, and the spatial predicates are - declared on the expression classes but their resolution metadata type is - designed by the per-operator port issues (#118, #119, #120). +* Table-shaped *reference slots* (DISJOIN ``this``/``reference`` and NEAREST + ``this``) resolve to a :class:`ResolvedRef`. NEAREST's ``reference`` slot — + whose accepted shapes are the non-table literal-range / column / + implicit-outer forms — resolves to a :class:`ResolvedInterval` (epic #114, + step 3). DISTANCE and the spatial predicates still declare their column / + literal slots but defer resolution to their port issues (#119, #120), which + reuse :class:`ResolvedInterval` and :class:`SlotDeferral`. """ from __future__ import annotations from dataclasses import dataclass +from dataclasses import field from dataclasses import replace from typing import Literal @@ -61,6 +66,7 @@ from giql.constants import DEFAULT_CHROM_COL from giql.constants import DEFAULT_END_COL from giql.constants import DEFAULT_START_COL +from giql.constants import DEFAULT_STRAND_COL from giql.expressions import Contains from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance @@ -69,13 +75,17 @@ from giql.expressions import SlotSpec from giql.expressions import SpatialSetPredicate from giql.expressions import Within +from giql.range_parser import RangeParser from giql.table import Table from giql.table import Tables __all__ = [ "META_KEY", "RefKind", + "IntervalKind", "ResolvedRef", + "ResolvedInterval", + "SlotDeferral", "OperatorResolution", "ResolutionError", "resolve_operator_refs", @@ -91,6 +101,14 @@ #: subquery trichotomy that ``sqlglot``'s ``scope.sources`` distinguishes. RefKind = Literal["registered_table", "cte", "subquery"] +#: The kinds a resolved *interval* slot can take. Unlike a :data:`RefKind`, an +#: interval slot does not resolve to a whole relation but to a column-qualified +#: genomic interval (``column`` / ``implicit_outer``) or a parsed literal range +#: (``literal_range``). These are the non-table shapes NEAREST's ``reference`` +#: slot accepts; DISTANCE and the spatial predicates (#119/#120) resolve the +#: same shapes onto the same :class:`ResolvedInterval` type. +IntervalKind = Literal["literal_range", "column", "implicit_outer"] + #: Canonical default genomic column names. A CTE or subquery reference is #: assumed (in the existing generator and here) to expose canonical #: ``chrom`` / ``start`` / ``end`` columns; validating that contract is a later @@ -158,6 +176,96 @@ class ResolvedRef: coverage_skippable: bool +@dataclass(frozen=True, slots=True) +class ResolvedInterval: + """Resolved metadata for one non-table interval operator slot. + + Where :class:`ResolvedRef` resolves a slot to a whole relation, a + ``ResolvedInterval`` resolves a slot to a single genomic interval expressed + as SQL fragments — either column references qualified by a relation alias + (``column`` / ``implicit_outer``) or the literal endpoints of a parsed + genomic range (``literal_range``). NEAREST's ``reference`` slot is the first + consumer; DISTANCE's two operands and the spatial predicates' column / range + slots (#119, #120) resolve onto this same type. + + Coordinate canonicalization deliberately stays in the emitter for now + (epic #114 step 8 / #123). The ``start`` / ``end`` fields therefore carry + the *raw, un-canonicalized* column SQL for the ``column`` / + ``implicit_outer`` kinds — the emitter wraps them with + :func:`giql.canonical.canonical_start` / ``canonical_end`` using + :attr:`table`. For the ``literal_range`` kind the endpoints are already + canonical 0-based half-open integer literals (parsed via + :meth:`~giql.range_parser.ParsedRange.to_zero_based_half_open`) and + :attr:`table` is ``None``; the emitter uses them verbatim. + + Attributes + ---------- + kind : IntervalKind + Whether the interval came from a literal range, an explicit column + reference, or an implicit LATERAL outer relation. + chrom : str + SQL fragment for the chromosome — a quoted literal (``'chr1'``) for a + ``literal_range`` or a qualified column (``alias."chrom"``) otherwise. + start : str + SQL fragment for the start endpoint. Canonical literal for a + ``literal_range``; raw column SQL (to be canonicalized by the emitter) + otherwise. + end : str + SQL fragment for the end endpoint, mirroring :attr:`start`. + strand : str | None + SQL fragment for the strand — a quoted literal for a ``literal_range`` + (``None`` when the range carries no strand), a qualified column for an + explicit ``column`` reference (always present), or a qualified column + for an ``implicit_outer`` reference only when the outer table declares a + strand column (``None`` otherwise). The emitter reads it only in + stranded mode. + table : Table | None + The :class:`~giql.table.Table` config backing a ``column`` / + ``implicit_outer`` interval, carried so the emitter can canonicalize the + endpoints; ``None`` for a ``literal_range`` (already canonical) or when + the qualifier resolves to no registered table. + """ + + kind: IntervalKind + chrom: str + start: str + end: str + strand: str | None + table: Table | None + + +@dataclass(frozen=True, slots=True) +class SlotDeferral: + """Why a slot was left unresolved, so the emitter raises the right error. + + The resolver is behavior-preserving during the epic #114 migration: when a + slot cannot be resolved it is *deferred* and the generator re-raises its + historical diagnostic. Some of those diagnostics need information the + emitter can no longer recompute on its own (the resolver moved the relevant + scope/ancestor walk out of the generator). A ``SlotDeferral`` carries that + information forward. + + NEAREST's implicit-outer reference is the motivating case: distinguishing + "no outer relation found in the LATERAL context" from "outer relation found + but not registered" requires the ancestor walk that now lives in this pass, + so the resolver records which failure occurred and, for the latter, the + offending relation label. DISTANCE and the spatial predicates (#119/#120) + reuse this channel for their own deferred-shape diagnostics. + + Attributes + ---------- + reason : str + A stable machine token naming the deferral cause (e.g. + ``"implicit_outer_missing"``, ``"implicit_outer_unregistered"``). + detail : str | None + Optional supporting datum for the emitter's message (e.g. the outer + relation label for ``"implicit_outer_unregistered"``). + """ + + reason: str + detail: str | None = None + + @dataclass(frozen=True, slots=True) class OperatorResolution: """Resolution metadata attached to a single GIQL operator node. @@ -166,20 +274,32 @@ class OperatorResolution: ---------- operator : str The operator expression class name (e.g. ``"GIQLDisjoin"``). - slots : dict[str, ResolvedRef] + slots : dict[str, ResolvedRef | ResolvedInterval] Mapping from a slot's :attr:`~giql.expressions.SlotSpec.arg` key to its - resolved :class:`ResolvedRef`. Only successfully resolved reference - slots appear; a slot left out was either not a reference slot or could - not be resolved (and the generator will raise its existing error). + resolved metadata — a :class:`ResolvedRef` for a table-shaped reference + slot, or a :class:`ResolvedInterval` for an interval slot. Only + successfully resolved slots appear; a slot left out was either not a + resolvable slot or could not be resolved (and the generator raises its + existing error, possibly aided by :attr:`deferrals`). + deferrals : dict[str, SlotDeferral] + Mapping from a slot key to a :class:`SlotDeferral` recording why the + slot was deferred, when the emitter needs that context to raise the + historical diagnostic verbatim. Empty for slots that resolved or whose + deferral the emitter can reclassify unaided. """ operator: str - slots: dict[str, ResolvedRef] + slots: dict[str, ResolvedRef | ResolvedInterval] + deferrals: dict[str, SlotDeferral] = field(default_factory=dict) - def slot(self, arg: str) -> ResolvedRef | None: - """Return the resolved reference for slot *arg*, or ``None``.""" + def slot(self, arg: str) -> ResolvedRef | ResolvedInterval | None: + """Return the resolved metadata for slot *arg*, or ``None``.""" return self.slots.get(arg) + def deferral(self, arg: str) -> SlotDeferral | None: + """Return the deferral recorded for slot *arg*, or ``None``.""" + return self.deferrals.get(arg) + def resolve_operator_refs(expression: exp.Expression, tables: Tables) -> exp.Expression: """Attach resolution metadata to every GIQL operator in *expression*. @@ -244,7 +364,8 @@ def _resolve_operator( node: exp.Expression, tables: Tables, cte_names: frozenset[str] ) -> None: """Resolve *node*'s reference slots and attach an :class:`OperatorResolution`.""" - slots: dict[str, ResolvedRef] = {} + slots: dict[str, ResolvedRef | ResolvedInterval] = {} + deferrals: dict[str, SlotDeferral] = {} if isinstance(node, GIQLDisjoin): target_ref = _resolve_target(node.this, tables) @@ -258,11 +379,20 @@ def _resolve_operator( target_ref = _resolve_target(node.this, tables) if target_ref is not None: slots["this"] = target_ref + # The reference slot resolves independently of the target — a literal + # range parses without any registered table, and an implicit-outer + # reference resolves against the enclosing scope. The interval and the + # deferral are mutually exclusive (one is always ``None``). + interval, deferral = _resolve_nearest_reference(node, tables) + if interval is not None: + slots["reference"] = interval + if deferral is not None: + deferrals["reference"] = deferral # DISTANCE and the spatial predicates declare only column / literal slots, # whose resolution metadata is designed by their port issues; the pass # attaches an (empty-slot) resolution so every operator carries metadata. - node.meta[META_KEY] = OperatorResolution(type(node).__name__, slots) + node.meta[META_KEY] = OperatorResolution(type(node).__name__, slots, deferrals) def _target_name(target: exp.Expression) -> str: @@ -367,6 +497,237 @@ def _resolve_disjoin_reference( return None +def _resolve_nearest_reference( + node: GIQLNearest, tables: Tables +) -> tuple[ResolvedInterval | None, SlotDeferral | None]: + """Resolve a NEAREST ``reference`` slot to a :class:`ResolvedInterval`. + + Ported here (epic #114, step 3) from the generator's historical + ``_resolve_nearest_reference`` / ``_find_outer_table_in_lateral_join``. The + accepted shapes are declarative on :class:`giql.expressions.GIQLNearest`: + + * **literal range** — a quoted genomic-range string, parsed to canonical + 0-based half-open literal endpoints. + * **column reference** — an explicit ``alias.col`` interval, resolved + against the enclosing SELECT's ``FROM`` / ``JOIN`` aliases exactly as the + generator's ``select_sql`` builds ``_alias_to_table`` / ``_current_table``. + * **implicit LATERAL outer** — no reference given, resolved to the outer + relation of the enclosing ``CROSS JOIN LATERAL`` via the same ancestor + walk the generator used. + + Returns ``(interval, None)`` on success or ``(None, deferral)`` when the + slot cannot be resolved; the generator then re-raises its historical error + (a ``None`` deferral means the emitter reclassifies the failure unaided, as + for a literal range that fails to parse). + """ + reference = node.args.get("reference") + + if reference is None: + return _resolve_implicit_outer(node, tables) + + # Literal genomic-range string (e.g. 'chr1:1000-2000'). sqlglot parses it as + # a string Literal; the generator detected it by the leading quote on the + # rendered SQL. + if isinstance(reference, exp.Literal) and reference.is_string: + return _resolve_literal_range(reference.name) + + # Explicit column reference (e.g. peaks.interval). + if isinstance(reference, exp.Column): + return _resolve_column_interval(node, reference.table or None, tables) + + # Fallback for any other shape: mirror the generator's string-level + # classification on the rendered reference (quote prefix => literal, + # otherwise a possibly-qualified column). + rendered = reference.sql() + if rendered.startswith("'") or rendered.startswith('"'): + return _resolve_literal_range(rendered.strip("'\"")) + qualifier = rendered.rsplit(".", 1)[0] if "." in rendered else None + return _resolve_column_interval(node, qualifier, tables) + + +def _resolve_literal_range( + range_str: str, +) -> tuple[ResolvedInterval | None, SlotDeferral | None]: + """Resolve a literal genomic-range reference to canonical literal endpoints. + + Defers (``(None, None)``) when the range fails to parse; the generator + re-parses and raises the historical "Could not parse reference genomic + range" diagnostic with the original exception text. + """ + try: + parsed = RangeParser.parse(range_str).to_zero_based_half_open() + except Exception: + return None, None + strand = f"'{parsed.strand}'" if parsed.strand else None + return ( + ResolvedInterval( + kind="literal_range", + chrom=f"'{parsed.chromosome}'", + start=str(parsed.start), + end=str(parsed.end), + strand=strand, + table=None, + ), + None, + ) + + +def _resolve_column_interval( + node: GIQLNearest, qualifier: str | None, tables: Tables +) -> tuple[ResolvedInterval, None]: + """Resolve an explicit column reference to a column :class:`ResolvedInterval`. + + Mirrors the generator's ``_get_column_refs`` / ``_resolve_table`` for an + explicit reference: the *qualifier* (the column's relation alias) is kept + verbatim for output, while the backing :class:`~giql.table.Table` — and thus + the physical column names and coordinate system — is looked up by resolving + that alias through the enclosing SELECT's alias map. An unresolved qualifier + falls back to default column names and no table (so the emitter applies no + canonicalization), exactly as the generator did. The strand column is always + emitted for an explicit reference, matching ``include_strand=True``. + """ + table = _lookup_aliased_table(node, qualifier, tables) + chrom_col = table.chrom_col if table else DEFAULT_CHROM_COL + start_col = table.start_col if table else DEFAULT_START_COL + end_col = table.end_col if table else DEFAULT_END_COL + strand_col = table.strand_col if (table and table.strand_col) else DEFAULT_STRAND_COL + prefix = f"{qualifier}." if qualifier else "" + return ( + ResolvedInterval( + kind="column", + chrom=f'{prefix}"{chrom_col}"', + start=f'{prefix}"{start_col}"', + end=f'{prefix}"{end_col}"', + strand=f'{prefix}"{strand_col}"', + table=table, + ), + None, + ) + + +def _resolve_implicit_outer( + node: GIQLNearest, tables: Tables +) -> tuple[ResolvedInterval | None, SlotDeferral | None]: + """Resolve an implicit-outer reference from the enclosing LATERAL join. + + Mirrors the generator's ``_find_outer_table_in_lateral_join`` followed by + its outer-table column resolution: the outer relation's label (alias or + name) qualifies the emitted columns, while the backing table is resolved by + mapping that label through the enclosing SELECT's alias map. Unlike an + explicit reference, the strand column is emitted only when the outer table + declares one — preserving the generator's divergent strand handling between + the two paths. + + Defers with a :class:`SlotDeferral` when no outer relation is found + (``implicit_outer_missing``) or the outer relation is unregistered + (``implicit_outer_unregistered``, carrying the offending label); the + generator raises the matching historical diagnostic. + """ + outer_label = _find_outer_table(node) + if outer_label is None: + return None, SlotDeferral("implicit_outer_missing") + + alias_map, _current = _enclosing_alias_map(node) + actual_name = alias_map.get(outer_label, outer_label) + table = tables.get(actual_name) + if table is None: + return None, SlotDeferral("implicit_outer_unregistered", outer_label) + + strand = f'{outer_label}."{table.strand_col}"' if table.strand_col else None + return ( + ResolvedInterval( + kind="implicit_outer", + chrom=f'{outer_label}."{table.chrom_col}"', + start=f'{outer_label}."{table.start_col}"', + end=f'{outer_label}."{table.end_col}"', + strand=strand, + table=table, + ), + None, + ) + + +def _lookup_aliased_table( + node: GIQLNearest, qualifier: str | None, tables: Tables +) -> Table | None: + """Resolve the :class:`~giql.table.Table` backing a qualified column. + + Replicates the generator's ``_resolve_table`` / ``_resolve_table_name``: a + dotted reference's *qualifier* is mapped to a registered-table name through + the enclosing SELECT's alias map (with ``_current_table`` as the FROM-clause + fallback); an unqualified reference resolves to no table. + """ + if not qualifier: + return None + alias_map, current_table = _enclosing_alias_map(node) + table_name = alias_map.get(qualifier, current_table) + return tables.get(table_name) if table_name else None + + +def _enclosing_alias_map(node: exp.Expression) -> tuple[dict[str, str], str | None]: + """Build the alias->table map of *node*'s enclosing SELECT. + + Mirrors ``BaseGIQLGenerator.select_sql`` exactly: the FROM-clause table and + every JOIN-clause table contribute an ``(alias or name) -> name`` entry, and + the FROM-clause table name is the ``_current_table`` fallback. Only direct + ``exp.Table`` operands participate (a LATERAL/derived join contributes + nothing), matching the generator. CTE and derived-table aliasing is out of + scope here because NEAREST column references resolve against physical + relations, exactly as the generator's hand-rolled map did. + """ + select = node.parent_select + alias_to_table: dict[str, str] = {} + current_table: str | None = None + if isinstance(select, exp.Select): + from_ = select.args.get("from_") + if from_ is not None and isinstance(from_.this, exp.Table): + current_table = from_.this.name + if from_.this.alias: + alias_to_table[from_.this.alias] = current_table + else: + alias_to_table[current_table] = current_table + for join in select.args.get("joins") or []: + if isinstance(join.this, exp.Table): + name = join.this.name + if join.this.alias: + alias_to_table[join.this.alias] = name + else: + alias_to_table[name] = name + return alias_to_table, current_table + + +def _find_outer_table(node: exp.Expression) -> str | None: + """Find the outer relation label of *node*'s enclosing LATERAL join. + + Walks up from the NEAREST node through the enclosing ``exp.Lateral`` to the + ``exp.Join``, then reads the join-owning SELECT's FROM-clause relation, + returning its alias or name. A byte-for-byte port of the generator's + ``_find_outer_table_in_lateral_join``; returns ``None`` when no such outer + relation exists (e.g. a standalone NEAREST). + """ + current: exp.Expression | None = node + while current is not None: + parent = current.parent + if parent is None: + break + if isinstance(parent, exp.Lateral): + current = parent + continue + if isinstance(parent, exp.Join): + select = parent.parent + if isinstance(select, exp.Select): + from_ = select.args.get("from_") + if from_ is not None: + table_expr = from_.this + if isinstance(table_expr, exp.Table): + return table_expr.alias or table_expr.name + elif isinstance(table_expr, exp.Alias): + return table_expr.alias + break + current = parent + return None + + def validate_operator_refs(expression: exp.Expression) -> None: """Assert every GIQL operator carries well-formed resolution metadata. @@ -407,14 +768,41 @@ def validate_operator_refs(expression: exp.Expression) -> None: specs: tuple[SlotSpec, ...] = getattr(node, "GIQL_SLOTS", ()) for spec in specs: - if not spec.is_ref_slot: - continue - ref = resolution.slots.get(spec.arg) - if ref is None: - # Deferred: unresolved reference slots are handled by the - # generator on its existing path in step 1. + resolved = resolution.slots.get(spec.arg) + if resolved is None: + # Deferred: an unresolved slot is handled by the generator on + # its existing path (and may carry a SlotDeferral). continue - _validate_ref(ref, spec, type(node).__name__) + if spec.is_ref_slot: + _validate_ref(resolved, spec, type(node).__name__) + else: + _validate_interval(resolved, spec, type(node).__name__) + + +def _validate_interval(interval: object, spec: SlotSpec, operator: str) -> None: + """Assert a single resolved interval is well-formed against its slot spec.""" + if not isinstance(interval, ResolvedInterval): + raise ResolutionError( + f"{operator} slot {spec.arg!r} carries {type(interval).__name__}, " + "expected ResolvedInterval." + ) + if interval.kind not in spec.accepts: + raise ResolutionError( + f"{operator} slot {spec.arg!r} resolved to kind {interval.kind!r}, " + f"which is not accepted by the slot (accepts {sorted(spec.accepts)})." + ) + if not all( + isinstance(part, str) for part in (interval.chrom, interval.start, interval.end) + ): + raise ResolutionError( + f"{operator} slot {spec.arg!r} has malformed interval endpoints; " + "expected SQL fragment strings for chrom/start/end." + ) + if interval.kind == "literal_range" and interval.table is not None: + raise ResolutionError( + f"{operator} slot {spec.arg!r} resolved to a literal_range but " + "carries a Table config; literal ranges are already canonical." + ) def _validate_ref(ref: object, spec: SlotSpec, operator: str) -> None: From eb07548251ffeeaf1afce6f7ffab372a8b3788b2 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 10:25:30 -0400 Subject: [PATCH 040/142] test: Pin NEAREST reference resolution across slot forms Cover literal-range, stranded-literal, column, and implicit-outer reference resolution, deferral of unresolvable references, parse failures, and interval-slot validation arms. Rewrite the outer-table regression to exercise the real LATERAL path instead of monkeypatching the deleted generator helper, and pin the strand divergence between explicit column references and implicit outer references. --- tests/generators/test_base.py | 118 +++++++----- tests/test_resolver.py | 329 +++++++++++++++++++++++++++++++++- 2 files changed, 402 insertions(+), 45 deletions(-) diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index 722ac36..9cd6598 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -566,6 +566,31 @@ def test_giqlnearest_sql_stranded(self, tables_with_peaks_and_genes): ) assert output == expected + def test_giqlnearest_sql_implicit_outer_without_strand_column(self): + """ + GIVEN a stranded NEAREST whose implicit-outer table declares no strand + column + WHEN the query is generated + THEN no strand filtering is emitted, preserving the divergence between + the explicit-column and implicit-outer reference paths. + """ + # Arrange + tables = Tables() + tables.register("nostr", Table("nostr", strand_col=None)) + tables.register("genes", Table("genes")) + sql = ( + "SELECT * FROM nostr " + "CROSS JOIN LATERAL NEAREST(genes, k := 1, stranded := true)" + ) + ast = parse_one(sql, dialect=GIQLDialect) + + # Act + output = BaseGIQLGenerator(tables=tables).generate(ast) + + # Assert + assert "strand" not in output + assert 'nostr."chrom" = genes."chrom"' in output + def test_giqlnearest_sql_signed(self, tables_with_peaks_and_genes): """ GIVEN a GIQLNearest with signed := true @@ -914,8 +939,7 @@ def test_giqlnearest_should_not_apply_gap_plus_one_for_closed_intervals( tables = Tables() tables.register("genes_closed", Table("genes_closed", interval_type="closed")) sql = ( - "SELECT * FROM NEAREST(" - "genes_closed, reference := 'chr1:1000-2000', k := 3)" + "SELECT * FROM NEAREST(genes_closed, reference := 'chr1:1000-2000', k := 3)" ) ast = parse_one(sql, dialect=GIQLDialect) generator = BaseGIQLGenerator(tables=tables) @@ -981,24 +1005,22 @@ def test_giqlnearest_sql_missing_outer_table_error( def test_giqlnearest_sql_outer_table_not_in_tables(self): """ - GIVEN a GIQLNearest in correlated mode where outer table is not registered - WHEN giqlnearest_sql is called + GIVEN a NEAREST whose implicit-outer relation is found but not registered + WHEN the query is generated THEN ValueError is raised listing the issue. """ tables = Tables() tables.register("genes", Table("genes")) - nearest = GIQLNearest( - this=exp.Table(this=exp.Identifier(this="genes")), - k=exp.Literal.number(3), - ) + # The outer relation (unknown_table) is present in the LATERAL context + # but is not a registered table, so the implicit-outer reference defers. + sql = "SELECT * FROM unknown_table CROSS JOIN LATERAL NEAREST(genes, k := 3)" + ast = parse_one(sql, dialect=GIQLDialect) generator = BaseGIQLGenerator(tables=tables) - generator._alias_to_table = {"unknown_table": "unknown_table"} - generator._find_outer_table_in_lateral_join = lambda x: "unknown_table" with pytest.raises(ValueError, match="not found in tables"): - generator.giqlnearest_sql(nearest) + generator.generate(ast) def test_giqlnearest_sql_invalid_reference_range(self, tables_with_peaks_and_genes): """ @@ -1261,8 +1283,7 @@ def test_intersects_should_canonicalize_table_columns_for_each_convention( # Assert expected = ( - "SELECT * FROM variants WHERE " - f"(\"chrom\" = 'chr1' AND {expected_predicate})" + f"SELECT * FROM variants WHERE (\"chrom\" = 'chr1' AND {expected_predicate})" ) assert output == expected @@ -1318,8 +1339,7 @@ def test_contains_should_canonicalize_table_columns_for_each_convention( # Assert expected = ( - "SELECT * FROM variants WHERE " - f"(\"chrom\" = 'chr1' AND {expected_predicate})" + f"SELECT * FROM variants WHERE (\"chrom\" = 'chr1' AND {expected_predicate})" ) assert output == expected @@ -1375,8 +1395,7 @@ def test_within_should_canonicalize_table_columns_for_each_convention( # Assert expected = ( - "SELECT * FROM variants WHERE " - f"(\"chrom\" = 'chr1' AND {expected_predicate})" + f"SELECT * FROM variants WHERE (\"chrom\" = 'chr1' AND {expected_predicate})" ) assert output == expected @@ -1545,24 +1564,39 @@ def test_intersects_any_should_canonicalize_disjuncts_when_table_is_one_based_cl "coordinate_system, interval_type, start_a, end_a, start_b, end_b", [ pytest.param( - "0based", "half_open", - 'a."start"', 'a."end"', 'b."start"', 'b."end"', + "0based", + "half_open", + 'a."start"', + 'a."end"', + 'b."start"', + 'b."end"', id="0based-half_open", ), pytest.param( - "0based", "closed", - 'a."start"', '(a."end" + 1)', 'b."start"', '(b."end" + 1)', + "0based", + "closed", + 'a."start"', + '(a."end" + 1)', + 'b."start"', + '(b."end" + 1)', id="0based-closed", ), pytest.param( - "1based", "half_open", - '(a."start" - 1)', '(a."end" - 1)', - '(b."start" - 1)', '(b."end" - 1)', + "1based", + "half_open", + '(a."start" - 1)', + '(a."end" - 1)', + '(b."start" - 1)', + '(b."end" - 1)', id="1based-half_open", ), pytest.param( - "1based", "closed", - '(a."start" - 1)', 'a."end"', '(b."start" - 1)', 'b."end"', + "1based", + "closed", + '(a."start" - 1)', + 'a."end"', + '(b."start" - 1)', + 'b."end"', id="1based-closed", ), ], @@ -1660,23 +1694,31 @@ def test_giqldistance_should_canonicalize_each_side_when_conventions_differ( "coordinate_system, interval_type, target_start, target_end", [ pytest.param( - "0based", "half_open", - 'genes."start"', 'genes."end"', + "0based", + "half_open", + 'genes."start"', + 'genes."end"', id="0based-half_open", ), pytest.param( - "0based", "closed", - 'genes."start"', '(genes."end" + 1)', + "0based", + "closed", + 'genes."start"', + '(genes."end" + 1)', id="0based-closed", ), pytest.param( - "1based", "half_open", - '(genes."start" - 1)', '(genes."end" - 1)', + "1based", + "half_open", + '(genes."start" - 1)', + '(genes."end" - 1)', id="1based-half_open", ), pytest.param( - "1based", "closed", - '(genes."start" - 1)', 'genes."end"', + "1based", + "closed", + '(genes."start" - 1)', + 'genes."end"', id="1based-closed", ), ], @@ -1715,9 +1757,7 @@ def test_giqlnearest_should_canonicalize_target_columns_for_each_convention( # Assert — distance CASE expression uses canonicalized target endpoints # against the (already-canonical) literal reference [1000, 2000). - assert ( - f"WHEN 1000 < {target_end} AND 2000 > {target_start} THEN 0" in output - ) + assert f"WHEN 1000 < {target_end} AND 2000 > {target_start} THEN 0" in output assert f"WHEN 2000 <= {target_start} THEN ({target_start} - 2000)" in output assert f"ELSE (1000 - {target_end})" in output @@ -1769,9 +1809,7 @@ def test_giqlnearest_should_canonicalize_outer_table_columns_when_reference_is_i leaving end raw — while leaving bed_a's target columns raw. """ # Arrange - sql = ( - "SELECT * FROM vcf_b CROSS JOIN LATERAL NEAREST(bed_a, k := 1)" - ) + sql = "SELECT * FROM vcf_b CROSS JOIN LATERAL NEAREST(bed_a, k := 1)" ast = parse_one(sql, dialect=GIQLDialect) generator = BaseGIQLGenerator(tables=tables_mixed_conventions) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index faf53dd..ab50a28 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -18,6 +18,7 @@ from giql.resolver import META_KEY from giql.resolver import OperatorResolution from giql.resolver import ResolutionError +from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef from giql.resolver import resolve_operator_refs from giql.resolver import validate_operator_refs @@ -47,6 +48,11 @@ def _disjoin_node(ast: exp.Expression) -> GIQLDisjoin: return next(n for n in ast.walk() if isinstance(n, GIQLDisjoin)) +def _nearest_node(ast: exp.Expression) -> GIQLNearest: + """Return the single GIQLNearest node reachable from an annotated AST.""" + return next(n for n in ast.walk() if isinstance(n, GIQLNearest)) + + class TestResolveOperatorRefs: """Tests for the resolve_operator_refs pass.""" @@ -278,6 +284,226 @@ def test_resolve_operator_refs_resolves_nearest_target(self): assert ref.kind == "registered_table" assert ref.name == "genes" + def test_resolve_operator_refs_resolves_nearest_literal_reference(self): + """Test that a NEAREST literal-range reference resolves to canonical literals. + + Given: + A standalone NEAREST with a literal genomic-range reference. + When: + Running the resolve pass. + Then: + It should attach a literal_range ResolvedInterval carrying quoted + chrom and canonical 0-based half-open endpoint literals and no Table. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("genes")) + + # Assert + ref = _nearest_node(ast).meta[META_KEY].slot("reference") + assert ref == ResolvedInterval( + kind="literal_range", + chrom="'chr1'", + start="1000", + end="2000", + strand=None, + table=None, + ) + + def test_resolve_operator_refs_resolves_nearest_stranded_literal(self): + """Test that a stranded literal-range reference carries its strand literal. + + Given: + A standalone NEAREST whose literal reference encodes a strand. + When: + Running the resolve pass. + Then: + It should attach a literal_range ResolvedInterval carrying the strand + as a quoted literal. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000:+', k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("genes")) + + # Assert + ref = _nearest_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "literal_range" + assert ref.strand == "'+'" + + def test_resolve_operator_refs_defers_unparseable_literal_reference(self): + """Test that an unparseable literal reference is deferred without a record. + + Given: + A standalone NEAREST whose literal reference cannot be parsed. + When: + Running the resolve pass. + Then: + It should leave the reference slot unresolved with no deferral + record, so the generator re-parses and raises its historical error. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, reference := 'invalid_range', k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("genes")) + + # Assert + resolution = _nearest_node(ast).meta[META_KEY] + assert resolution.slot("reference") is None + assert resolution.deferral("reference") is None + + def test_resolve_operator_refs_resolves_nearest_column_reference(self): + """Test that an explicit column reference resolves to qualified columns. + + Given: + A correlated NEAREST whose reference is an aliased outer column + (``p.interval`` where ``p`` aliases a registered table). + When: + Running the resolve pass. + Then: + It should attach a column ResolvedInterval whose endpoints keep the + alias verbatim and whose Table config comes from the aliased table. + """ + # Arrange + ast = parse_one( + "SELECT * FROM peaks AS p " + "CROSS JOIN LATERAL NEAREST(genes, reference := p.interval, k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("peaks", "genes")) + + # Assert + ref = _nearest_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "column" + assert ref.chrom == 'p."chrom"' + assert ref.start == 'p."start"' + assert ref.end == 'p."end"' + assert ref.strand == 'p."strand"' + assert ref.table is not None and ref.table.name == "peaks" + + def test_resolve_operator_refs_resolves_implicit_outer_reference(self): + """Test that an omitted reference resolves to the LATERAL outer relation. + + Given: + A correlated NEAREST with no reference inside a CROSS JOIN LATERAL + over a registered outer table. + When: + Running the resolve pass. + Then: + It should attach an implicit_outer ResolvedInterval qualified by the + outer relation and backed by the outer table's config. + """ + # Arrange + ast = parse_one( + "SELECT * FROM peaks CROSS JOIN LATERAL NEAREST(genes, k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("peaks", "genes")) + + # Assert + ref = _nearest_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "implicit_outer" + assert ref.chrom == 'peaks."chrom"' + assert ref.strand == 'peaks."strand"' + assert ref.table is not None and ref.table.name == "peaks" + + def test_resolve_operator_refs_implicit_outer_omits_strand_without_column(self): + """Test that an implicit-outer reference omits strand when the table has none. + + Given: + A correlated NEAREST over an outer table configured without a strand + column. + When: + Running the resolve pass. + Then: + It should attach an implicit_outer ResolvedInterval whose strand is + None, preserving the generator's divergent strand handling. + """ + # Arrange + tables = Tables() + tables.register("nostr", Table("nostr", strand_col=None)) + tables.register("genes", Table("genes")) + ast = parse_one( + "SELECT * FROM nostr CROSS JOIN LATERAL NEAREST(genes, k := 1)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + ref = _nearest_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "implicit_outer" + assert ref.strand is None + + def test_resolve_operator_refs_defers_implicit_outer_missing(self): + """Test that a missing LATERAL outer relation records a deferral. + + Given: + A NEAREST with no reference and no enclosing LATERAL outer relation. + When: + Running the resolve pass. + Then: + It should leave the reference unresolved and record an + implicit_outer_missing deferral for the generator's error. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("genes")) + + # Assert + resolution = _nearest_node(ast).meta[META_KEY] + assert resolution.slot("reference") is None + assert resolution.deferral("reference").reason == "implicit_outer_missing" + + def test_resolve_operator_refs_defers_implicit_outer_unregistered(self): + """Test that an unregistered LATERAL outer relation records its label. + + Given: + A NEAREST with no reference whose enclosing LATERAL outer relation is + found but not registered. + When: + Running the resolve pass. + Then: + It should record an implicit_outer_unregistered deferral carrying the + offending outer-relation label. + """ + # Arrange + ast = parse_one( + "SELECT * FROM unknown_table CROSS JOIN LATERAL NEAREST(genes, k := 3)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("genes")) + + # Assert + deferral = _nearest_node(ast).meta[META_KEY].deferral("reference") + assert deferral.reason == "implicit_outer_unregistered" + assert deferral.detail == "unknown_table" + def test_resolve_operator_refs_returns_same_expression(self): """Test that the pass annotates and returns the same AST in place. @@ -372,15 +598,17 @@ def test_resolve_operator_refs_annotates_distance_and_set_predicate(self): assert predicate.meta[META_KEY].slots == {} def test_resolve_operator_refs_unregistered_nearest_target_left_unresolved(self): - """Test that an unregistered NEAREST target leaves the slot unresolved. + """Test that an unregistered NEAREST target leaves the target slot unresolved. Given: - A NEAREST whose target table is not registered. + A NEAREST whose target table is not registered (its literal-range + reference resolves independently of the target). When: Running the resolve pass. Then: - It should attach an OperatorResolution with no resolved slots, - deferring the error to the generator. + It should attach an OperatorResolution whose target slot is + unresolved, deferring the target error to the generator, while the + literal reference still resolves. """ # Arrange ast = parse_one( @@ -393,7 +621,8 @@ def test_resolve_operator_refs_unregistered_nearest_target_left_unresolved(self) # Assert nearest = next(n for n in ast.walk() if isinstance(n, GIQLNearest)) - assert nearest.meta[META_KEY].slots == {} + assert nearest.meta[META_KEY].slot("this") is None + assert nearest.meta[META_KEY].slot("reference").kind == "literal_range" @settings(max_examples=50) @given(names=st.lists(_identifiers, min_size=4, max_size=4, unique=True)) @@ -615,3 +844,93 @@ def test_validate_operator_refs_rejects_non_resolvedref_slot_value(self): # Act & assert with pytest.raises(ResolutionError, match="expected ResolvedRef"): validate_operator_refs(ast) + + def test_validate_operator_refs_accepts_resolved_interval(self): + """Test that validation accepts a well-formed interval slot. + + Given: + A NEAREST annotated by the resolve pass with a resolved interval + reference. + When: + Running validation over it. + Then: + It should not raise. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)", + dialect=GIQLDialect, + ) + resolve_operator_refs(ast, _tables("genes")) + + # Act & assert + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_disallowed_interval_kind(self): + """Test that validation rejects an interval slot of a disallowed kind. + + Given: + A NEAREST reference slot annotated with a ResolvedInterval whose kind + (a table-shaped ``registered_table``) the slot does not accept. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the rejected kind. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)", + dialect=GIQLDialect, + ) + node = _nearest_node(ast) + node.meta[META_KEY] = OperatorResolution( + "GIQLNearest", + { + "reference": ResolvedInterval( + kind="registered_table", + chrom="'chr1'", + start="1000", + end="2000", + strand=None, + table=None, + ) + }, + ) + + # Act & assert + with pytest.raises(ResolutionError, match="not accepted"): + validate_operator_refs(ast) + + def test_validate_operator_refs_rejects_non_resolvedinterval_slot_value(self): + """Test that validation rejects an interval slot holding the wrong type. + + Given: + A NEAREST reference slot whose resolution metadata holds a + ResolvedRef instead of a ResolvedInterval. + When: + Running validation over the tree. + Then: + It should raise a ResolutionError naming the expected type. + """ + # Arrange + ast = parse_one( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)", + dialect=GIQLDialect, + ) + node = _nearest_node(ast) + node.meta[META_KEY] = OperatorResolution( + "GIQLNearest", + { + "reference": ResolvedRef( + kind="registered_table", + name="genes", + cols=("chrom", "start", "end"), + table=Table("genes"), + coverage_skippable=False, + ) + }, + ) + + # Act & assert + with pytest.raises(ResolutionError, match="expected ResolvedInterval"): + validate_operator_refs(ast) From b65c1793df6678a9c2c3a55b764e77fbd681d127 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 11:36:53 -0400 Subject: [PATCH 041/142] refactor: Port DISTANCE emitter to resolver pass metadata Resolve DISTANCE's two column operands in the ResolveOperatorRefs pass and read them from the attached metadata at emission time. A new ResolvedColumn dataclass carries the alias-qualified physical chrom, start, end, and strand fragments plus the backing Table config, stored in a columns channel on OperatorResolution alongside the existing slot and deferral channels. Alias-to-table derivation reuses the shared enclosing-alias-map helper introduced by the NEAREST port, so one implementation serves both operators, and the validation boundary gains a column arm. Coordinate canonicalization stays in the emitter, wrapping the raw endpoint fragments with the Table config the metadata carries, until the CanonicalizeCoordinates migration retires it. The emitter falls back to the legacy string-level resolution when no metadata is attached, because the DISTANCE suites invoke the generator directly without the pipeline; the shared legacy helpers remain live for the spatial predicate paths. --- src/giql/generators/base.py | 109 +++++++++++------- src/giql/resolver.py | 214 +++++++++++++++++++++++++++++++++--- 2 files changed, 266 insertions(+), 57 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 631d8dc..b7b5e58 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -20,6 +20,7 @@ from giql.range_parser import RangeParser from giql.resolver import META_KEY from giql.resolver import OperatorResolution +from giql.resolver import ResolvedColumn from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef from giql.resolver import resolve_operator_refs @@ -374,64 +375,45 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: def giqldistance_sql(self, expression: GIQLDistance) -> str: """Generate SQL CASE expression for DISTANCE function. + Reads the :class:`~giql.resolver.ResolvedColumn` metadata that + ``ResolveOperatorRefs`` (pass 1) attaches to each interval operand. When + the pass deferred an operand (a literal range, an unqualified column, or + a tree the pass never reached) the emitter falls back to its historical + string-level resolution and raises the same diagnostics as before. + + Coordinate canonicalization stays here (epic #114 step 8 / issue #123): + the resolved metadata carries each operand's :class:`~giql.table.Table` + so the endpoints are wrapped identically. + :param expression: GIQLDistance expression node :return: SQL CASE expression string calculating genomic distance """ - # Extract the two interval arguments - interval_a = expression.this - interval_b = expression.args.get("expression") - stranded = self._extract_bool_param(expression.args.get("stranded")) signed = self._extract_bool_param(expression.args.get("signed")) - # Get SQL representations - interval_a_sql = self.sql(interval_a) - interval_b_sql = self.sql(interval_b) + col_a = self._distance_operand(expression, "this", "first") + col_b = self._distance_operand(expression, "expression", "second") - # Check if we're dealing with column-to-column or column-to-literal - if "." in interval_a_sql and not interval_a_sql.startswith("'"): - # Column reference for interval_a - if stranded: - chrom_a, start_a, end_a, strand_a = self._get_column_refs( - interval_a_sql, None, include_strand=True - ) - else: - chrom_a, start_a, end_a = self._get_column_refs(interval_a_sql, None) - strand_a = None - else: - # Literal range - not implemented yet for interval_a - raise ValueError("Literal range as first argument not yet supported") - - if "." in interval_b_sql and not interval_b_sql.startswith("'"): - # Column reference for interval_b - if stranded: - chrom_b, start_b, end_b, strand_b = self._get_column_refs( - interval_b_sql, None, include_strand=True - ) - else: - chrom_b, start_b, end_b = self._get_column_refs(interval_b_sql, None) - strand_b = None - else: - # Literal range - not implemented yet - raise ValueError("Literal range as second argument not yet supported") + # Strand columns are consumed only in stranded mode (matching the + # historical 3-tuple vs 4-tuple branching in the legacy emitter). + strand_a = col_a.strand if stranded else None + strand_b = col_b.strand if stranded else None # Distance math below assumes 0-based half-open. - table_a = self._resolve_table(interval_a_sql) - table_b = self._resolve_table(interval_b_sql) - start_a = canonical_start(start_a, table_a) - end_a = canonical_end(end_a, table_a) - start_b = canonical_start(start_b, table_b) - end_b = canonical_end(end_b, table_b) + start_a = canonical_start(col_a.start, col_a.table) + end_a = canonical_end(col_a.end, col_a.table) + start_b = canonical_start(col_b.start, col_b.table) + end_b = canonical_end(col_b.end, col_b.table) # Generate CASE expression return self._generate_distance_case( - chrom_a, + col_a.chrom, start_a, end_a, strand_a, - chrom_b, + col_b.chrom, start_b, end_b, strand_b, @@ -439,6 +421,51 @@ def giqldistance_sql(self, expression: GIQLDistance) -> str: signed=signed, ) + def _distance_operand( + self, expression: GIQLDistance, arg: str, position: str + ) -> ResolvedColumn: + """Resolve one DISTANCE interval operand to a :class:`ResolvedColumn`. + + Prefers the metadata attached by ``ResolveOperatorRefs`` (pass 1). When + the pass deferred the operand — it could not resolve a literal range or + an unqualified column, or never ran (the generator was invoked directly + without the pass) — this falls back to the legacy ``_get_column_refs`` / + ``_resolve_table`` path, raising the historical literal-range error for + a non-column operand. + + :param expression: + GIQLDistance expression node + :param arg: + The operand arg key (``"this"`` or ``"expression"``) + :param position: + Human-readable operand position for the error message (``"first"`` + or ``"second"``) + :return: + The resolved column operand + :raises ValueError: + If the operand is a literal range rather than a column reference + """ + resolution = expression.meta.get(META_KEY) + if isinstance(resolution, OperatorResolution): + resolved = resolution.column(arg) + if resolved is not None: + return resolved + + # Deferred: fall back to string-level resolution. + operand_sql = self.sql(expression.args.get(arg)) + if "." in operand_sql and not operand_sql.startswith("'"): + chrom, start, end, strand = self._get_column_refs( + operand_sql, None, include_strand=True + ) + return ResolvedColumn( + chrom=chrom, + start=start, + end=end, + strand=strand, + table=self._resolve_table(operand_sql), + ) + raise ValueError(f"Literal range as {position} argument not yet supported") + def _generate_distance_case( self, chrom_a: str, diff --git a/src/giql/resolver.py b/src/giql/resolver.py index c2fac29..f7fffbc 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -47,9 +47,11 @@ ``this``) resolve to a :class:`ResolvedRef`. NEAREST's ``reference`` slot — whose accepted shapes are the non-table literal-range / column / implicit-outer forms — resolves to a :class:`ResolvedInterval` (epic #114, - step 3). DISTANCE and the spatial predicates still declare their column / - literal slots but defer resolution to their port issues (#119, #120), which - reuse :class:`ResolvedInterval` and :class:`SlotDeferral`. + step 3). DISTANCE's two *column* operands (``this`` / ``expression``) are + resolved to a :class:`ResolvedColumn` and attached through the separate + :attr:`OperatorResolution.columns` channel (epic #114, step 4). The spatial + predicates still declare their column / literal slots but defer resolution to + their port issue (#120), which reuses these resolved-metadata types. """ from __future__ import annotations @@ -85,6 +87,7 @@ "IntervalKind", "ResolvedRef", "ResolvedInterval", + "ResolvedColumn", "SlotDeferral", "OperatorResolution", "ResolutionError", @@ -234,6 +237,52 @@ class ResolvedInterval: table: Table | None +@dataclass(frozen=True, slots=True) +class ResolvedColumn: + """Resolved metadata for one column-shaped interval operand. + + Models the resolution of a DISTANCE operand (``a.interval``) — an + ``exp.Column`` qualified by a table alias — into the physical genomic + columns it references, qualified by that alias. Unlike a + :class:`ResolvedRef` (which names a whole relation), a column operand + resolves to concrete SQL column expressions ready to drop into the + emitter's distance arithmetic. + + Coordinate canonicalization stays in the emitter (epic #114 defers its + removal to step 8 / issue #123); this metadata therefore carries the + backing :class:`~giql.table.Table` so the emitter keeps wrapping the + endpoints with :func:`giql.canonical.canonical_start` / + :func:`giql.canonical.canonical_end` exactly as before. + + Attributes + ---------- + chrom : str + The chromosome column qualified by the operand's alias, e.g. + ``a."chrom"``. + start : str + The start column qualified by the operand's alias, e.g. ``a."start"``. + end : str + The end column qualified by the operand's alias, e.g. ``a."end"``. + strand : str | None + The strand column qualified by the operand's alias, e.g. + ``a."strand"``. Always resolved when the operand is a column (mirroring + the generator's ``_get_column_refs(..., include_strand=True)``); the + emitter consumes it only in stranded mode. + table : Table | None + The :class:`~giql.table.Table` config backing the operand's relation + (carrying its coordinate system), or ``None`` when the alias does not + resolve to a registered table (an unregistered relation is assumed + canonical, exactly as the generator's ``_resolve_table`` returns + ``None``). + """ + + chrom: str + start: str + end: str + strand: str | None + table: Table | None + + @dataclass(frozen=True, slots=True) class SlotDeferral: """Why a slot was left unresolved, so the emitter raises the right error. @@ -286,11 +335,17 @@ class OperatorResolution: slot was deferred, when the emitter needs that context to raise the historical diagnostic verbatim. Empty for slots that resolved or whose deferral the emitter can reclassify unaided. + columns : dict[str, ResolvedColumn] + Mapping from a *column* operand's arg key to its resolved + :class:`ResolvedColumn`. Carries DISTANCE's two interval operands; an + operand the pass could not resolve (a literal range, or an unqualified + column) is left out and the generator raises its existing error. """ operator: str slots: dict[str, ResolvedRef | ResolvedInterval] deferrals: dict[str, SlotDeferral] = field(default_factory=dict) + columns: dict[str, ResolvedColumn] = field(default_factory=dict) def slot(self, arg: str) -> ResolvedRef | ResolvedInterval | None: """Return the resolved metadata for slot *arg*, or ``None``.""" @@ -300,6 +355,10 @@ def deferral(self, arg: str) -> SlotDeferral | None: """Return the deferral recorded for slot *arg*, or ``None``.""" return self.deferrals.get(arg) + def column(self, arg: str) -> ResolvedColumn | None: + """Return the resolved column operand for *arg*, or ``None``.""" + return self.columns.get(arg) + def resolve_operator_refs(expression: exp.Expression, tables: Tables) -> exp.Expression: """Attach resolution metadata to every GIQL operator in *expression*. @@ -335,8 +394,8 @@ def resolve_operator_refs(expression: exp.Expression, tables: Tables) -> exp.Exp # Fallback for any operator a scope walk did not reach (e.g. if scope # construction failed). Resolving with no visible CTE names keeps the pass - # behavior-preserving: a missed CTE reference simply stays unresolved and - # the generator handles it on its existing path. + # behavior-preserving: a missed CTE reference or column operand simply stays + # unresolved and the generator handles it on its existing path. for node in expression.walk(): if isinstance(node, _OPERATORS) and id(node) not in seen: seen.add(id(node)) @@ -363,9 +422,10 @@ def _safe_traverse_scope(expression: exp.Expression) -> list[Scope]: def _resolve_operator( node: exp.Expression, tables: Tables, cte_names: frozenset[str] ) -> None: - """Resolve *node*'s reference slots and attach an :class:`OperatorResolution`.""" + """Resolve *node*'s slots and attach an :class:`OperatorResolution`.""" slots: dict[str, ResolvedRef | ResolvedInterval] = {} deferrals: dict[str, SlotDeferral] = {} + columns: dict[str, ResolvedColumn] = {} if isinstance(node, GIQLDisjoin): target_ref = _resolve_target(node.this, tables) @@ -388,11 +448,15 @@ def _resolve_operator( slots["reference"] = interval if deferral is not None: deferrals["reference"] = deferral - # DISTANCE and the spatial predicates declare only column / literal slots, - # whose resolution metadata is designed by their port issues; the pass + elif isinstance(node, GIQLDistance): + columns = _resolve_distance_columns(node, tables) + # The spatial predicates declare only column / literal slots, whose + # resolution metadata is designed by their port issue (#120); the pass # attaches an (empty-slot) resolution so every operator carries metadata. - node.meta[META_KEY] = OperatorResolution(type(node).__name__, slots, deferrals) + node.meta[META_KEY] = OperatorResolution( + type(node).__name__, slots, deferrals, columns + ) def _target_name(target: exp.Expression) -> str: @@ -426,6 +490,92 @@ def _resolve_target(target: exp.Expression, tables: Tables) -> ResolvedRef | Non ) +def _resolve_distance_columns( + node: GIQLDistance, tables: Tables +) -> dict[str, ResolvedColumn]: + """Resolve DISTANCE's two interval operands to :class:`ResolvedColumn`\\s. + + DISTANCE's ``this`` and ``expression`` operands are both column refs (per + its :attr:`~giql.expressions.GIQLDistance.GIQL_SLOTS`). Each is resolved + against the operand's table alias, mirroring the generator's historical + ``_get_column_refs`` / ``_resolve_table`` path: the alias's physical + genomic columns (from the registered :class:`~giql.table.Table` config, or + the canonical defaults) qualified by that alias, plus the backing table + config for the emitter's canonicalization wrapping. + + An operand the pass cannot resolve — a literal range, or an unqualified + column — is omitted; the generator then raises its historical + "Literal range as ... argument not yet supported" error. + """ + alias_map, current_table = _enclosing_alias_map(node) + columns: dict[str, ResolvedColumn] = {} + for arg in ("this", "expression"): + operand = node.args.get(arg) + resolved = _resolve_column_operand(operand, tables, alias_map, current_table) + if resolved is not None: + columns[arg] = resolved + return columns + + +def _resolve_column_operand( + operand: exp.Expression | None, + tables: Tables, + alias_map: dict[str, str], + current_table: str | None, +) -> ResolvedColumn | None: + """Resolve a single column operand, or ``None`` if it is not a column ref. + + Mirrors the generator's ``_get_column_refs`` / ``_resolve_table`` exactly: + the operand's alias is resolved to an underlying table name via the alias + map (with the current FROM table as fallback), the physical column names + come from that table's config (or the canonical defaults), and every column + is qualified by the operand's alias. + + Returns ``None`` for an operand that is not a qualified column (a literal + range or an unaliased column), deferring the diagnostic to the generator. + """ + if not isinstance(operand, exp.Column): + return None + alias = operand.table + if not alias: + # An unqualified column has no alias to qualify by; the generator + # treats it as a literal range and raises its existing error. + return None + + table_name = alias_map.get(alias, current_table) + table = tables.get(table_name) if table_name else None + chrom_col, start_col, end_col, strand_col = _physical_cols(table) + return ResolvedColumn( + chrom=f'{alias}."{chrom_col}"', + start=f'{alias}."{start_col}"', + end=f'{alias}."{end_col}"', + strand=f'{alias}."{strand_col}"', + table=table, + ) + + +def _physical_cols(table: Table | None) -> tuple[str, str, str, str]: + """Return the ``(chrom, start, end, strand)`` physical column names. + + Mirrors ``_get_column_refs``: the canonical defaults unless the registered + :class:`~giql.table.Table` overrides them, with the strand column falling + back to the default when the table declares none. + """ + if table is None: + return ( + DEFAULT_CHROM_COL, + DEFAULT_START_COL, + DEFAULT_END_COL, + DEFAULT_STRAND_COL, + ) + return ( + table.chrom_col, + table.start_col, + table.end_col, + table.strand_col or DEFAULT_STRAND_COL, + ) + + def _resolve_disjoin_reference( reference: exp.Expression | None, target_ref: ResolvedRef, @@ -769,14 +919,19 @@ def validate_operator_refs(expression: exp.Expression) -> None: specs: tuple[SlotSpec, ...] = getattr(node, "GIQL_SLOTS", ()) for spec in specs: resolved = resolution.slots.get(spec.arg) - if resolved is None: - # Deferred: an unresolved slot is handled by the generator on - # its existing path (and may carry a SlotDeferral). + if resolved is not None: + if spec.is_ref_slot: + _validate_ref(resolved, spec, type(node).__name__) + else: + _validate_interval(resolved, spec, type(node).__name__) continue - if spec.is_ref_slot: - _validate_ref(resolved, spec, type(node).__name__) - else: - _validate_interval(resolved, spec, type(node).__name__) + # A column operand (DISTANCE) resolves through the separate columns + # channel rather than the slots map; validate it there. + column = resolution.columns.get(spec.arg) + if column is not None: + _validate_column(column, spec, type(node).__name__) + # Otherwise deferred: an unresolved slot is handled by the generator + # on its existing path (and may carry a SlotDeferral). def _validate_interval(interval: object, spec: SlotSpec, operator: str) -> None: @@ -805,6 +960,33 @@ def _validate_interval(interval: object, spec: SlotSpec, operator: str) -> None: ) +def _validate_column(column: object, spec: SlotSpec, operator: str) -> None: + """Assert a single resolved column operand is well-formed against its slot. + + DISTANCE's two interval operands resolve to a :class:`ResolvedColumn` in the + :attr:`OperatorResolution.columns` channel. A column operand is only ever + attached for a slot whose declared shapes include ``"column"``; the endpoint + fragments must be SQL strings ready to drop into the emitter's arithmetic. + """ + if not isinstance(column, ResolvedColumn): + raise ResolutionError( + f"{operator} slot {spec.arg!r} carries {type(column).__name__}, " + "expected ResolvedColumn." + ) + if "column" not in spec.accepts: + raise ResolutionError( + f"{operator} slot {spec.arg!r} resolved to a column operand, which is " + f"not accepted by the slot (accepts {sorted(spec.accepts)})." + ) + if not all( + isinstance(part, str) for part in (column.chrom, column.start, column.end) + ): + raise ResolutionError( + f"{operator} slot {spec.arg!r} has malformed column endpoints; " + "expected SQL fragment strings for chrom/start/end." + ) + + def _validate_ref(ref: object, spec: SlotSpec, operator: str) -> None: """Assert a single resolved reference is well-formed against its slot spec.""" if not isinstance(ref, ResolvedRef): From 716e2ac44cb574f3846a9de1672327974043d2cb Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 11:36:54 -0400 Subject: [PATCH 042/142] test: Cover DISTANCE operand resolution and emitter equivalence Pin column-operand resolution across aliased default columns, custom column names, custom and missing strand columns, unregistered aliases, literal-range deferral, and unqualified-column deferral, plus the new column validation arm. Add a transpilation regression asserting the resolver-driven path emits byte-identical SQL to direct generation. --- tests/test_distance_transpilation.py | 20 +++ tests/test_resolver.py | 237 +++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) diff --git a/tests/test_distance_transpilation.py b/tests/test_distance_transpilation.py index 69b0b0c..380aa19 100644 --- a/tests/test_distance_transpilation.py +++ b/tests/test_distance_transpilation.py @@ -5,6 +5,7 @@ from sqlglot import parse_one +from giql import transpile from giql.dialect import GIQLDialect from giql.generators import BaseGIQLGenerator @@ -69,6 +70,25 @@ def test_distance_transpilation_postgres(self): assert output == expected, f"Expected:\n{expected}\n\nGot:\n{output}" + def test_distance_resolver_path_matches_direct_generation(self): + """ + GIVEN a DISTANCE query over registered default-convention tables + WHEN transpiling through the full pipeline (the resolver pass) versus + generating directly from the parsed AST + THEN both paths should emit byte-identical SQL, proving the + ResolvedColumn metadata path reproduces the legacy string path + """ + query = ( + "SELECT DISTANCE(a.interval, b.interval) AS dist " + "FROM features_a a, features_b b" + ) + + via_transpile = transpile(query, tables=["features_a", "features_b"]) + ast = parse_one(query, dialect=GIQLDialect) + via_generate = BaseGIQLGenerator().generate(ast) + + assert via_transpile == via_generate + def test_distance_transpilation_signed_duckdb(self): """ GIVEN a GIQL query with DISTANCE(..., signed := true) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index ab50a28..9f7b85a 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -18,6 +18,7 @@ from giql.resolver import META_KEY from giql.resolver import OperatorResolution from giql.resolver import ResolutionError +from giql.resolver import ResolvedColumn from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef from giql.resolver import resolve_operator_refs @@ -53,6 +54,11 @@ def _nearest_node(ast: exp.Expression) -> GIQLNearest: return next(n for n in ast.walk() if isinstance(n, GIQLNearest)) +def _distance_node(ast: exp.Expression) -> GIQLDistance: + """Return the single GIQLDistance node reachable from an annotated AST.""" + return next(n for n in ast.walk() if isinstance(n, GIQLDistance)) + + class TestResolveOperatorRefs: """Tests for the resolve_operator_refs pass.""" @@ -656,6 +662,237 @@ def test_resolve_operator_refs_with_arbitrary_table_config(self, names): validate_operator_refs(ast) +class TestResolveDistanceColumns: + """Tests for DISTANCE interval-operand (column) resolution.""" + + def test_resolve_distance_columns_resolves_aliased_operands(self): + """Test that DISTANCE operands resolve to alias-qualified default columns. + + Given: + A DISTANCE over two aliased registered tables with default columns. + When: + Running the resolve pass. + Then: + It should attach a ResolvedColumn per operand, qualified by the + operand alias and backed by the registered Table config. + """ + # Arrange + tables = _tables("intervals_a", "intervals_b") + ast = parse_one( + "SELECT DISTANCE(a.interval, b.interval) AS dist " + "FROM intervals_a a, intervals_b b", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + resolution = _distance_node(ast).meta[META_KEY] + assert resolution.column("this") == ResolvedColumn( + chrom='a."chrom"', + start='a."start"', + end='a."end"', + strand='a."strand"', + table=tables.get("intervals_a"), + ) + assert resolution.column("expression") == ResolvedColumn( + chrom='b."chrom"', + start='b."start"', + end='b."end"', + strand='b."strand"', + table=tables.get("intervals_b"), + ) + + def test_resolve_distance_columns_honors_custom_column_names(self): + """Test that DISTANCE operands pick up a table's custom column names. + + Given: + A DISTANCE whose operand's registered table declares custom + chrom/start/end column names. + When: + Running the resolve pass. + Then: + The resolved column should carry the custom physical names, + qualified by the operand alias. + """ + # Arrange + tables = Tables() + tables.register( + "features_a", + Table("features_a", chrom_col="chr", start_col="lo", end_col="hi"), + ) + tables.register("features_b", Table("features_b")) + ast = parse_one( + "SELECT DISTANCE(a.interval, b.interval) FROM features_a a, features_b b", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + col = _distance_node(ast).meta[META_KEY].column("this") + assert col.chrom == 'a."chr"' + assert col.start == 'a."lo"' + assert col.end == 'a."hi"' + assert col.table is tables.get("features_a") + + def test_resolve_distance_columns_resolves_custom_strand_column(self): + """Test that a DISTANCE operand resolves a table's custom strand column. + + Given: + A DISTANCE whose operand's table declares a custom strand column. + When: + Running the resolve pass. + Then: + The resolved column's strand member should name that custom column, + qualified by the operand alias. + """ + # Arrange + tables = Tables() + tables.register( + "features_a", + Table("features_a", strand_col="dir"), + ) + tables.register("features_b", Table("features_b")) + ast = parse_one( + "SELECT DISTANCE(a.interval, b.interval) FROM features_a a, features_b b", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + col = _distance_node(ast).meta[META_KEY].column("this") + assert col.strand == 'a."dir"' + + def test_resolve_distance_columns_defaults_when_table_strandless(self): + """Test that a strandless table still yields a default strand column. + + Given: + A DISTANCE operand whose table declares no strand column. + When: + Running the resolve pass. + Then: + The resolved column's strand should fall back to the default + strand name, mirroring the generator's _get_column_refs. + """ + # Arrange + tables = Tables() + tables.register("features_a", Table("features_a", strand_col=None)) + tables.register("features_b", Table("features_b")) + ast = parse_one( + "SELECT DISTANCE(a.interval, b.interval) FROM features_a a, features_b b", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + col = _distance_node(ast).meta[META_KEY].column("this") + assert col.strand == 'a."strand"' + + def test_resolve_distance_columns_unregistered_alias_has_no_table(self): + """Test that an operand over an unregistered relation carries no Table. + + Given: + A DISTANCE over subquery-derived relations that are not registered + tables. + When: + Running the resolve pass. + Then: + The resolved column should carry default column names and a None + Table config, so the emitter applies no canonicalization. + """ + # Arrange + ast = parse_one( + "SELECT DISTANCE(a.interval, b.interval) FROM " + "(SELECT 1) AS a, (SELECT 1) AS b", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, Tables()) + + # Assert + col = _distance_node(ast).meta[META_KEY].column("this") + assert col.chrom == 'a."chrom"' + assert col.table is None + + def test_resolve_distance_columns_defers_literal_range_operand(self): + """Test that a literal-range operand is left unresolved. + + Given: + A DISTANCE whose second operand is a literal genomic range string. + When: + Running the resolve pass. + Then: + It should resolve the column operand but leave the literal operand + unresolved, deferring the diagnostic to the generator. + """ + # Arrange + ast = parse_one( + "SELECT DISTANCE(a.interval, 'chr1:1000-2000') FROM features_a a", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features_a")) + + # Assert + resolution = _distance_node(ast).meta[META_KEY] + assert resolution.column("this") is not None + assert resolution.column("expression") is None + + def test_resolve_distance_columns_defers_unqualified_column(self): + """Test that an unqualified column operand is left unresolved. + + Given: + A DISTANCE whose first operand is a column with no table qualifier. + When: + Running the resolve pass. + Then: + It should leave that operand unresolved, deferring to the generator + (which treats an unqualified operand as a literal range). + """ + # Arrange + ast = parse_one( + "SELECT DISTANCE(interval, b.interval) FROM features_a a, features_b b", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features_a", "features_b")) + + # Assert + resolution = _distance_node(ast).meta[META_KEY] + assert resolution.column("this") is None + assert resolution.column("expression") is not None + + def test_resolve_distance_columns_pass_validates(self): + """Test that a DISTANCE-annotated tree passes the validation boundary. + + Given: + A DISTANCE annotated with resolved column metadata. + When: + Running the validation boundary. + Then: + It should not raise (column operands are not ref slots). + """ + # Arrange + ast = parse_one( + "SELECT DISTANCE(a.interval, b.interval) FROM features_a a, features_b b", + dialect=GIQLDialect, + ) + resolve_operator_refs(ast, _tables("features_a", "features_b")) + + # Act & assert + validate_operator_refs(ast) + + class TestValidateOperatorRefs: """Tests for the validate_operator_refs validation boundary.""" From e79093acf79f66282029056f1e63540ba99d744d Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 12:27:39 -0400 Subject: [PATCH 043/142] refactor: Port spatial predicate emitters to resolver pass metadata MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve the INTERSECTS, CONTAINS, WITHIN, and set-predicate column operands in the ResolveOperatorRefs pass and read them from the attached metadata at emission time. Operands resolve into the existing columns channel as ResolvedColumn entries through a predicate-specific precedence layer: literal-range and set predicates anchor the left operand to the enclosing FROM table, unqualified operands format without an alias while resolving config through that table, and column-to-column shapes resolve both operands through the alias map only — mirroring the generator's historical current-table semantics exactly. The shared alias-map and physical-column helpers from the NEAREST and DISTANCE ports are reused. The emitters consume metadata through a new predicate-operand reader that falls back to the legacy string-level resolution when no metadata is attached, since the generator unit suites invoke generate() directly. The legacy helpers and select_sql alias tracking remain live for that fallback; their retirement is the final-audit step. --- src/giql/generators/base.py | 100 +++++++++++++++++-------- src/giql/resolver.py | 144 ++++++++++++++++++++++++++++++++++-- 2 files changed, 207 insertions(+), 37 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index b7b5e58..189a14e 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -557,6 +557,42 @@ def _generate_distance_case( f"ELSE ({start_a} - {end_b}) END END" ) + def _predicate_operand( + self, expression: exp.Expression, arg: str, ctx_table: str | None + ) -> ResolvedColumn: + """Return the :class:`ResolvedColumn` for a spatial predicate operand. + + Reads the column resolution attached to *expression* by the + ``ResolveOperatorRefs`` pass (the metadata-driven path used by the full + transpile pipeline). When the pass did not annotate the node — e.g. a + generator invoked on a bare AST without running pass 1 — it falls back to + the generator's historical ``_current_table`` / alias-map resolution so + direct ``generate()`` callers keep their existing behavior. Both paths + format physical column references identically, so the emitted SQL is the + same regardless of which produced the :class:`ResolvedColumn`. + + :param expression: + The spatial predicate node carrying the resolution metadata. + :param arg: + The operand slot key (``"this"`` or ``"expression"``). + :param ctx_table: + The current-table resolution context for the fallback path — + ``self._current_table`` for a literal-range operand, ``None`` for a + column-to-column operand. + :return: + The resolved column metadata. + """ + resolution = expression.meta.get(META_KEY) + if isinstance(resolution, OperatorResolution): + resolved = resolution.column(arg) + if resolved is not None: + return resolved + + column_ref = self.sql(expression, arg) + chrom, start, end = self._get_column_refs(column_ref, ctx_table) + table = self._resolve_table(column_ref, ctx_table) + return ResolvedColumn(chrom=chrom, start=start, end=end, strand="", table=table) + def _generate_spatial_op(self, expression: exp.Binary, op_type: str) -> str: """Generate SQL for a spatial operation. @@ -567,18 +603,20 @@ def _generate_spatial_op(self, expression: exp.Binary, op_type: str) -> str: :return: SQL predicate string """ - left = self.sql(expression, "this") right_raw = self.sql(expression, "expression") # Check if right side is a column reference or a literal range string if "." in right_raw and not right_raw.startswith("'"): # Column-to-column join (e.g., a.interval INTERSECTS b.interval) - return self._generate_column_join(left, right_raw, op_type) + left = self._predicate_operand(expression, "this", None) + right = self._predicate_operand(expression, "expression", None) + return self._generate_column_join(left, right, op_type) else: # Literal range string (e.g., interval INTERSECTS 'chr1:1000-2000') try: range_str = right_raw.strip("'\"") parsed_range = RangeParser.parse(range_str).to_zero_based_half_open() + left = self._predicate_operand(expression, "this", self._current_table) return self._generate_range_predicate(left, parsed_range, op_type) except Exception as e: raise ValueError( @@ -587,14 +625,15 @@ def _generate_spatial_op(self, expression: exp.Binary, op_type: str) -> str: def _generate_range_predicate( self, - column_ref: str, + column: ResolvedColumn, parsed_range: ParsedRange, op_type: str, ) -> str: """Generate SQL predicate for a range operation. - :param column_ref: - Column reference (e.g., 'v.interval' or 'interval') + :param column: + Resolved column operand (physical chrom/start/end fragments plus the + backing :class:`~giql.table.Table` config for canonicalization). :param parsed_range: Parsed genomic range :param op_type: @@ -602,13 +641,12 @@ def _generate_range_predicate( :return: SQL predicate string """ - # Get column references - chrom_col, raw_start_col, raw_end_col = self._get_column_refs( - column_ref, self._current_table - ) - table = self._resolve_table(column_ref, self._current_table) - start_col = canonical_start(raw_start_col, table) - end_col = canonical_end(raw_end_col, table) + # Canonicalize the raw physical endpoints to 0-based half-open. The + # alias-qualified column fragments come pre-resolved on the + # ResolvedColumn; canonicalization stays here (epic #114 step #123). + chrom_col = column.chrom + start_col = canonical_start(column.start, column.table) + end_col = canonical_end(column.end, column.table) chrom = parsed_range.chromosome start = parsed_range.start @@ -648,28 +686,28 @@ def _generate_range_predicate( raise ValueError(f"Unknown operation: {op_type}") - def _generate_column_join(self, left_col: str, right_col: str, op_type: str) -> str: + def _generate_column_join( + self, left: ResolvedColumn, right: ResolvedColumn, op_type: str + ) -> str: """Generate SQL for column-to-column spatial joins. - :param left_col: - Left column reference (e.g., 'a.interval') - :param right_col: - Right column reference (e.g., 'b.interval') + :param left: + Resolved left column operand (e.g., for 'a.interval'). + :param right: + Resolved right column operand (e.g., for 'b.interval'). :param op_type: 'intersects', 'contains', or 'within' :return: SQL predicate string """ - # Get column references for both sides - # Pass None to let _get_column_refs extract and resolve table from column ref - l_chrom, raw_l_start, raw_l_end = self._get_column_refs(left_col, None) - r_chrom, raw_r_start, raw_r_end = self._get_column_refs(right_col, None) - l_table = self._resolve_table(left_col) - r_table = self._resolve_table(right_col) - l_start = canonical_start(raw_l_start, l_table) - l_end = canonical_end(raw_l_end, l_table) - r_start = canonical_start(raw_r_start, r_table) - r_end = canonical_end(raw_r_end, r_table) + # Canonicalize each side's raw physical endpoints; the alias-qualified + # chrom/start/end fragments come pre-resolved on the ResolvedColumns. + l_chrom = left.chrom + r_chrom = right.chrom + l_start = canonical_start(left.start, left.table) + l_end = canonical_end(left.end, left.table) + r_start = canonical_start(right.start, right.table) + r_end = canonical_end(right.end, right.table) if op_type == "intersects": # Ranges overlap if: chrom1 = chrom2 AND start1 < end2 AND end1 > start2 @@ -709,11 +747,15 @@ def _generate_spatial_set(self, expression: SpatialSetPredicate) -> str: :return: SQL predicate string """ - column_ref = self.sql(expression, "this") operator = expression.args["operator"] quantifier = expression.args["quantifier"] ranges = expression.args["ranges"] + # Resolve the (single) left column operand once; every range condition + # compares against the same column. The set predicate's ranges are + # always literals, so only this operand needs resolution. + column = self._predicate_operand(expression, "this", self._current_table) + # Parse all ranges parsed_ranges = [] for range_expr in ranges: @@ -726,7 +768,7 @@ def _generate_spatial_set(self, expression: SpatialSetPredicate) -> str: # Generate conditions for each range conditions = [] for parsed_range in parsed_ranges: - condition = self._generate_range_predicate(column_ref, parsed_range, op_type) + condition = self._generate_range_predicate(column, parsed_range, op_type) conditions.append(condition) # Combine with AND (for ALL) or OR (for ANY) diff --git a/src/giql/resolver.py b/src/giql/resolver.py index f7fffbc..2c6889a 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -50,8 +50,12 @@ step 3). DISTANCE's two *column* operands (``this`` / ``expression``) are resolved to a :class:`ResolvedColumn` and attached through the separate :attr:`OperatorResolution.columns` channel (epic #114, step 4). The spatial - predicates still declare their column / literal slots but defer resolution to - their port issue (#120), which reuses these resolved-metadata types. + predicates' column operands (:class:`~giql.expressions.Intersects` / + ``Contains`` / ``Within`` ``this`` and column-shaped ``expression``, and + ``SpatialSetPredicate`` ``this``) resolve onto the same + :class:`ResolvedColumn` type through that columns channel (epic #114, + step #120); a literal-range ``expression`` slot stays deferred and the emitter + parses it on its existing path. """ from __future__ import annotations @@ -337,9 +341,11 @@ class OperatorResolution: deferral the emitter can reclassify unaided. columns : dict[str, ResolvedColumn] Mapping from a *column* operand's arg key to its resolved - :class:`ResolvedColumn`. Carries DISTANCE's two interval operands; an - operand the pass could not resolve (a literal range, or an unqualified - column) is left out and the generator raises its existing error. + :class:`ResolvedColumn`. Carries DISTANCE's two interval operands and + the spatial predicates' column operands; an operand the pass could not + resolve (a literal range, or an unqualified column outside a + current-table context) is left out and the generator raises its existing + error. """ operator: str @@ -450,9 +456,8 @@ def _resolve_operator( deferrals["reference"] = deferral elif isinstance(node, GIQLDistance): columns = _resolve_distance_columns(node, tables) - # The spatial predicates declare only column / literal slots, whose - # resolution metadata is designed by their port issue (#120); the pass - # attaches an (empty-slot) resolution so every operator carries metadata. + elif isinstance(node, (Intersects, Contains, Within, SpatialSetPredicate)): + columns = _resolve_predicate_columns(node, tables) node.meta[META_KEY] = OperatorResolution( type(node).__name__, slots, deferrals, columns @@ -517,6 +522,129 @@ def _resolve_distance_columns( return columns +def _resolve_predicate_columns( + node: exp.Expression, tables: Tables +) -> dict[str, ResolvedColumn]: + """Resolve a spatial predicate's column operands to :class:`ResolvedColumn`. + + Mirrors the generator's historical ``_generate_spatial_op`` operand handling + exactly so the emitted SQL is byte-identical: + + * A literal-range predicate (``column INTERSECTS 'chr1:...'``) and every + ``SpatialSetPredicate`` resolve only their left ``this`` operand, against + the FROM-clause current table — matching ``_generate_range_predicate``, + which passes ``self._current_table`` as the resolution context. + * A column-to-column predicate (``a.interval CONTAINS b.interval``) resolves + both ``this`` and ``expression`` with *no* current-table context, so each + operand resolves through the alias map — matching ``_generate_column_join``, + which passes ``None`` as the context for both sides. + + Most column-to-column ``INTERSECTS`` joins never reach here: the + :class:`~giql.transformer.IntersectsBinnedJoinTransformer` rewrites them into + binned equi-joins before this pass runs, deleting the ``Intersects`` node. + Column-to-column ``CONTAINS`` / ``WITHIN`` (which that transformer leaves + untouched) and non-join ``INTERSECTS`` shapes still reach the emitter, so the + column-join branch is exercised. + + Reuses the shared ``_enclosing_alias_map`` (FROM/JOIN alias derivation) and + ``_physical_cols`` helpers; the predicate-specific bit lives in + :func:`_resolve_predicate_column`, whose current-table-vs-alias-only + precedence differs from DISTANCE's ``_resolve_column_operand``. + """ + alias_map, current_table = _enclosing_alias_map(node) + this_node = node.this + + if isinstance(node, SpatialSetPredicate): + if not isinstance(this_node, exp.Column): + return {} + return { + "this": _resolve_predicate_column( + this_node, current_table, alias_map, current_table, tables + ) + } + + right = node.args.get("expression") + if isinstance(right, exp.Column) and right.table: + # Column-to-column: resolve both operands with no current-table context, + # so each resolves through the alias map. + columns: dict[str, ResolvedColumn] = {} + if isinstance(this_node, exp.Column): + columns["this"] = _resolve_predicate_column( + this_node, None, alias_map, current_table, tables + ) + columns["expression"] = _resolve_predicate_column( + right, None, alias_map, current_table, tables + ) + return columns + + # Literal-range predicate: resolve only the left operand, anchored to the + # FROM-clause current table. + if not isinstance(this_node, exp.Column): + return {} + return { + "this": _resolve_predicate_column( + this_node, current_table, alias_map, current_table, tables + ) + } + + +def _resolve_predicate_column( + column: exp.Column, + ctx_table: str | None, + alias_map: dict[str, str], + current_table: str | None, + tables: Tables, +) -> ResolvedColumn: + """Resolve one spatial-predicate column operand to a :class:`ResolvedColumn`. + + Replicates the generator's ``_get_column_refs`` / ``_resolve_table`` (and the + ``_resolve_table_name`` precedence underneath them) exactly: + + * the operand's table qualifier is kept verbatim for output formatting; + * the *config* table name is resolved by precedence — an explicit + ``ctx_table`` (the caller-supplied current table) wins; otherwise a + qualified operand resolves through the alias map (falling back to + ``current_table``), and an unqualified operand resolves to no table; + * physical column names come from the resolved :class:`~giql.table.Table` + config via the shared ``_physical_cols`` helper, or the canonical defaults + when no table backs the operand. + + This is the minimal predicate-specific layer DISTANCE's + ``_resolve_column_operand`` cannot serve: it formats unqualified operands + (bare ``interval``) and lets the literal-range path anchor to ``ctx_table`` + directly, where ``_resolve_column_operand`` requires a qualifier and always + routes through the alias map. + """ + alias = column.table or "" + qualified = bool(alias) + + if ctx_table: + table_name: str | None = ctx_table + elif qualified: + table_name = alias_map.get(alias, current_table) + else: + table_name = None + + table = tables.get(table_name) if table_name else None + chrom_col, start_col, end_col, strand_col = _physical_cols(table) + + if qualified: + return ResolvedColumn( + chrom=f'{alias}."{chrom_col}"', + start=f'{alias}."{start_col}"', + end=f'{alias}."{end_col}"', + strand=f'{alias}."{strand_col}"', + table=table, + ) + return ResolvedColumn( + chrom=f'"{chrom_col}"', + start=f'"{start_col}"', + end=f'"{end_col}"', + strand=f'"{strand_col}"', + table=table, + ) + + def _resolve_column_operand( operand: exp.Expression | None, tables: Tables, From 0e1ebb2bc32898c8d559c67dd93ddfb42056bcb4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 12:27:39 -0400 Subject: [PATCH 044/142] test: Pin spatial predicate operand resolution Cover qualified and unqualified literal-range predicates, custom column names, column-to-column operand pairs, ANY and ALL set predicates, unregistered-table defaults, and deferral when no base FROM table exists. --- tests/test_resolver.py | 225 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index 9f7b85a..c92afae 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -11,10 +11,13 @@ from sqlglot import parse_one from giql.dialect import GIQLDialect +from giql.expressions import Contains from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance from giql.expressions import GIQLNearest +from giql.expressions import Intersects from giql.expressions import SpatialSetPredicate +from giql.expressions import Within from giql.resolver import META_KEY from giql.resolver import OperatorResolution from giql.resolver import ResolutionError @@ -59,6 +62,11 @@ def _distance_node(ast: exp.Expression) -> GIQLDistance: return next(n for n in ast.walk() if isinstance(n, GIQLDistance)) +def _predicate_node(ast: exp.Expression, node_type) -> exp.Expression: + """Return the single predicate node of *node_type* reachable from the AST.""" + return next(n for n in ast.walk() if isinstance(n, node_type)) + + class TestResolveOperatorRefs: """Tests for the resolve_operator_refs pass.""" @@ -893,6 +901,223 @@ def test_resolve_distance_columns_pass_validates(self): validate_operator_refs(ast) +class TestResolvePredicateColumns: + """Tests for spatial-predicate column-operand resolution.""" + + def test_resolve_aliased_literal_predicate_qualifies_columns(self): + """Test that an aliased literal-range predicate qualifies its column. + + Given: + A literal-range INTERSECTS over an aliased FROM table. + When: + Running the resolve pass. + Then: + It should resolve the left operand to alias-qualified physical + columns carrying the table config, and attach no expression column. + """ + # Arrange + ast = parse_one( + "SELECT * FROM variants AS v WHERE v.interval INTERSECTS 'chr1:1000-2000'", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("variants")) + + # Assert + node = _predicate_node(ast, Intersects) + column = node.meta[META_KEY].column("this") + assert column == ResolvedColumn( + chrom='v."chrom"', + start='v."start"', + end='v."end"', + strand='v."strand"', + table=_tables("variants").get("variants"), + ) + assert node.meta[META_KEY].column("expression") is None + + def test_resolve_undotted_predicate_uses_current_table(self): + """Test that an undotted predicate operand resolves to the FROM table. + + Given: + A literal-range CONTAINS whose left operand is the bare genomic + pseudo-column over a single-table FROM clause. + When: + Running the resolve pass. + Then: + It should resolve the operand to unqualified physical columns backed + by the FROM-clause table config. + """ + # Arrange + ast = parse_one( + "SELECT * FROM peaks WHERE interval CONTAINS 'chr1:1500'", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("peaks")) + + # Assert + column = _predicate_node(ast, Contains).meta[META_KEY].column("this") + assert column.chrom == '"chrom"' + assert column.start == '"start"' + assert column.end == '"end"' + assert column.table.name == "peaks" + + def test_resolve_custom_columns_in_predicate(self): + """Test that a predicate operand carries the table's custom column names. + + Given: + A literal-range WITHIN over a table configured with custom genomic + column names. + When: + Running the resolve pass. + Then: + It should resolve the operand fragments to the custom physical + columns. + """ + # Arrange + tables = Tables() + tables.register( + "peaks", + Table( + "peaks", + chrom_col="chromosome", + start_col="start_pos", + end_col="end_pos", + strand_col="sense", + ), + ) + ast = parse_one( + "SELECT * FROM peaks WHERE interval WITHIN 'chr1:1000-2000'", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, tables) + + # Assert + column = _predicate_node(ast, Within).meta[META_KEY].column("this") + assert column.chrom == '"chromosome"' + assert column.start == '"start_pos"' + assert column.end == '"end_pos"' + assert column.strand == '"sense"' + + def test_resolve_column_to_column_predicate_resolves_both_operands(self): + """Test that a column-to-column predicate resolves both operands. + + Given: + A column-to-column WITHIN joining two aliased tables (a shape the + binned-join transformer leaves untouched, so it reaches the pass). + When: + Running the resolve pass. + Then: + It should resolve both operands through the alias map to their + respective table configs. + """ + # Arrange + ast = parse_one( + "SELECT a.name, b.name FROM intervals_a a, intervals_b b " + "WHERE a.interval WITHIN b.interval", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("intervals_a", "intervals_b")) + + # Assert + resolution = _predicate_node(ast, Within).meta[META_KEY] + this_col = resolution.column("this") + expr_col = resolution.column("expression") + assert this_col.chrom == 'a."chrom"' + assert this_col.table.name == "intervals_a" + assert expr_col.chrom == 'b."chrom"' + assert expr_col.table.name == "intervals_b" + + def test_resolve_set_predicate_resolves_left_operand(self): + """Test that a set predicate resolves only its left column operand. + + Given: + An INTERSECTS ANY set predicate over a single FROM table. + When: + Running the resolve pass. + Then: + It should resolve the left operand to the FROM table's columns and + attach no per-range column metadata. + """ + # Arrange + ast = parse_one( + "SELECT * FROM peaks " + "WHERE interval INTERSECTS ANY('chr1:1000-2000', 'chr2:500-1000')", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("peaks")) + + # Assert + resolution = _predicate_node(ast, SpatialSetPredicate).meta[META_KEY] + assert resolution.column("this").chrom == '"chrom"' + assert set(resolution.columns) == {"this"} + + def test_resolve_predicate_over_unregistered_table_uses_defaults(self): + """Test that a predicate over an unregistered table defers to defaults. + + Given: + A literal-range INTERSECTS over a FROM table that is not registered. + When: + Running the resolve pass with no table configs. + Then: + It should resolve the operand to canonical default columns with no + backing table config. + """ + # Arrange + ast = parse_one( + "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, Tables()) + + # Assert + column = _predicate_node(ast, Intersects).meta[META_KEY].column("this") + assert column == ResolvedColumn( + chrom='"chrom"', + start='"start"', + end='"end"', + strand='"strand"', + table=None, + ) + + def test_resolve_predicate_without_base_from_table_defers(self): + """Test that an undotted predicate with no base FROM table resolves to defaults. + + Given: + A literal-range INTERSECTS in a SELECT whose FROM clause is a derived + table, so the enclosing scope exposes no base FROM table. + When: + Running the resolve pass. + Then: + It should resolve the undotted operand to unqualified default + columns with no backing table config. + """ + # Arrange + ast = parse_one( + "SELECT * FROM (SELECT * FROM peaks) AS d " + "WHERE interval INTERSECTS 'chr1:1000-2000'", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("peaks")) + + # Assert + column = _predicate_node(ast, Intersects).meta[META_KEY].column("this") + assert column.chrom == '"chrom"' + assert column.table is None + + class TestValidateOperatorRefs: """Tests for the validate_operator_refs validation boundary.""" From 82a8298c9436eb50bda8902f2fa2cf868c59cde0 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 15:08:43 -0400 Subject: [PATCH 045/142] refactor: Drive DISJOIN coordinate canonicalization through pass 2 Opt DISJOIN into the CanonicalizeCoordinates pass by setting GIQL_CANONICALIZE on its expression class, and consume the pass output in giqldisjoin_sql instead of wrapping endpoints inline. The cut arithmetic, breakpoints, and coverage filter now read already-canonical columns; the four input-side canonical_start and canonical_end calls are gone from the emitter. Because DISJOIN passes whole target rows through and synthesizes its disjoin_start and disjoin_end output columns as text at generation time, the canonical wrapper projects the full row via SELECT * REPLACE rather than the interval triple, and the pass records each wrapped slot's original Table encoding on the resolution. The emitter reads that encoding to round-trip the passed-through interval and the output columns back to the target convention. Canonical 0-based half-open targets take an identity fast path with no wrapper and byte-identical SQL; non-canonical encodings gain a wrapper CTE. --- src/giql/canonicalizer.py | 109 +++++++++++++++++++++++--------- src/giql/expressions.py | 9 +++ src/giql/generators/base.py | 120 +++++++++++++++++++++++++++++------- src/giql/resolver.py | 11 ++++ 4 files changed, 197 insertions(+), 52 deletions(-) diff --git a/src/giql/canonicalizer.py b/src/giql/canonicalizer.py index bb0e34f..714a007 100644 --- a/src/giql/canonicalizer.py +++ b/src/giql/canonicalizer.py @@ -34,6 +34,20 @@ tradeoff the epic calls out (only synthesize a wrapper when canonicalization actually changes columns). +Engine portability (known limitation) +------------------------------------- +The wrapper projection uses ``SELECT * REPLACE (...)`` to canonicalize the +interval columns in place while passing every other source column through +untouched (the registry declares only the genomic columns, so an explicit +full-column projection is not available). ``* REPLACE`` is supported by DuckDB, +BigQuery, Snowflake, and ClickHouse, but **not** by PostgreSQL, SQLite, or +DataFusion — so a non-canonical encoding currently transpiles to +engine-incompatible SQL on those targets. Identity-encoded (default 0-based +half-open) relations are unaffected: they skip wrapping entirely and emit +portable SQL. Making the emit strategy dialect-aware (an explicit portable +projection when the target lacks ``REPLACE`` or the full schema is declared) is +tracked in https://github.com/abdenlab/giql/issues/132. + Gating (epic #114, step 6) -------------------------- The pass is gated per operator by a ``GIQL_CANONICALIZE`` class attribute on the @@ -48,11 +62,17 @@ De-canonicalization hook ------------------------- -The outermost ``SELECT`` projection receives a de-canonicalization rewrite for -any output column that a migrated operator emitted in canonical form but that -must land in the user's preferred encoding. With no operator migrated in this -issue that rewrite has nothing to act on; :func:`_decanonicalize_outputs` is the -designed-but-inert hook the port issues will fill in. +A migrated operator's *output* columns must land back in the target relation's +declared encoding. Epic #114 step 6 envisioned a rewrite of the outermost +``SELECT`` projection, but that placement is wrong for a table function: DISJOIN +synthesizes its ``disjoin_*`` output and its passed-through interval at +*generation* time, so those columns do not exist as AST in this pass, and a +``SELECT *`` consumer hides them from any outer-projection rewrite. So +:func:`_decanonicalize_outputs` instead records each wrapped slot's *original* +:class:`~giql.table.Table` on the operator's +:class:`~giql.resolver.OperatorResolution`, and the operator's emitter reads it +to de-canonicalize those synthesized columns where it generates them (DISJOIN, +issue #122). """ from __future__ import annotations @@ -246,22 +266,39 @@ def _fresh_name(next_name, taken: set[str]) -> str: def _canonical_projection(ref: ResolvedRef) -> exp.Select: """Build the ``SELECT`` body that projects *ref*'s table to canonical form. - The projection exposes the canonical ``chrom`` / ``start`` / ``end`` columns - under their original physical names, with ``start`` / ``end`` rewritten by - the :mod:`giql.canonical` arithmetic for the table's declared encoding. This - is the interval contract every CTE / subquery reference is assumed to satisfy - (canonical 0-based half-open ``chrom`` / ``start`` / ``end``); operator port - issues #122 / #123 may extend it with pass-through columns as their emitters - require. + The projection is a **full-row passthrough**: ``SELECT *`` keeps every + physical column of the source relation, and a star ``REPLACE`` rewrites only + the two interval columns — ``start`` / ``end``, under their original physical + names — with the :mod:`giql.canonical` arithmetic for the table's declared + encoding. ``chrom`` and every non-interval column flow through the star + untouched. + + The full row (rather than a bare ``chrom`` / ``start`` / ``end`` triple) is + required by table-function operators whose final projection passes the whole + source row through — DISJOIN's ``SELECT t.*`` (#122) — and by their join-back + semantics, which key on the source's physical columns. A CTE / subquery + reference that only needs the canonical interval triple still reads those + three columns from the same wrapper. """ - chrom, start, end = ref.cols + _chrom, start, end = ref.cols table = ref.table relation = ref.name - return exp.select( - exp.alias_(exp.column(chrom), chrom), - exp.alias_(_canonical_start_expr(start, table), start), - exp.alias_(_canonical_end_expr(end, table), end), - ).from_(exp.to_table(relation)) + # Quote the interval identifiers: the canonical column names are physical and + # routinely reserved words (the default genomic layout's ``start`` / ``end``), + # so the executed wrapper must quote them. + star = exp.Star( + replace=[ + exp.alias_( + _canonical_start_expr(start, table), + exp.to_identifier(start, quoted=True), + ), + exp.alias_( + _canonical_end_expr(end, table), + exp.to_identifier(end, quoted=True), + ), + ] + ) + return exp.Select(expressions=[star]).from_(exp.to_table(relation)) def _canonical_start_expr(start: str, table: Table | None) -> exp.Expression: @@ -272,7 +309,7 @@ def _canonical_start_expr(start: str, table: Table | None) -> exp.Expression: - ``0based``: ``start`` (identity) - ``1based``: ``start - 1`` """ - col = exp.column(start) + col = exp.column(exp.to_identifier(start, quoted=True)) if table is None or table.coordinate_system == "0based": return col return exp.paren(exp.Sub(this=col, expression=exp.Literal.number(1))) @@ -288,7 +325,7 @@ def _canonical_end_expr(end: str, table: Table | None) -> exp.Expression: - ``1based`` / ``half_open``: ``end - 1`` - ``1based`` / ``closed``: ``end`` (identity) """ - col = exp.column(end) + col = exp.column(exp.to_identifier(end, quoted=True)) if table is None: return col key = (table.coordinate_system, table.interval_type) @@ -343,13 +380,27 @@ def _decanonicalize_outputs( expression: exp.Expression, targets: list[tuple[exp.Expression, str, ResolvedRef]], ) -> None: - """De-canonicalize migrated operator outputs in the outermost projection. - - Inert hook (epic #114, step 6). The outermost ``SELECT`` projection list - should rewrite any output column a migrated operator emitted in canonical - form back into the user's preferred encoding. No operator is migrated in - issue #121, so there is nothing to rewrite; the operator port issues (#122, - #123) fill this in alongside flipping their ``GIQL_CANONICALIZE`` flags. + """Preserve each wrapped slot's original encoding for the emitter's output. + + A wrapped slot's :class:`~giql.resolver.ResolvedRef` is rewritten to a + ``Table``-free canonical-CTE ref, which would otherwise lose the + (non-canonical) encoding the operator's *output* must round-trip back into. + + The de-canonicalization itself cannot be applied on the AST in this pass for + a table-function operator: DISJOIN synthesizes its ``disjoin_*`` columns and + its passed-through interval at *generation* time, so those columns do not + exist as AST here, and a ``SELECT *`` consumer hides them from any + outer-projection rewrite. The originally-envisioned outermost-projection + rewrite (epic #114, step 6) is therefore wrong for projected + table-function columns; instead this hook records the per-slot original + :class:`~giql.table.Table` on the :class:`~giql.resolver.OperatorResolution`, + and the operator's emitter reads it to de-canonicalize those synthesized + columns where it generates them (see :issue:`122`). + + *targets* carries the original (pre-rewrite) refs, so ``ref.table`` is the + source relation's declared encoding. """ - # Intentionally empty until an operator opts in (see module docstring). - return None + for node, arg, ref in targets: + resolution = node.meta.get(META_KEY) + if isinstance(resolution, OperatorResolution): + resolution.output_tables[arg] = ref.table diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 4364fce..52e5fce 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -336,6 +336,15 @@ class GIQLDisjoin(exp.Func): "reference": False, # Optional: reference table/CTE name or subquery } + #: Opt DISJOIN into the CanonicalizeCoordinates pass (epic #114 step 7, + #: issue #122). With this flag set, pass 2 wraps every non-canonical + #: interval-bearing operand in a canonical ``__giql_canon_*`` CTE and + #: rewrites the slot to point at it, so the emitter consumes already-canonical + #: 0-based half-open columns instead of canonicalizing inline. Identity + #: (0-based half-open) operands are left unwrapped and the emitted SQL stays + #: byte-identical. + GIQL_CANONICALIZE = True + GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), SlotSpec( diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 189a14e..2bbf918 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -280,41 +280,56 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: filter drops sub-intervals overlapping no reference interval. When no ``reference`` is given it defaults to the target set. - Coordinate-system round-tripping is handled by - :func:`giql.canonical.decanonical_start` / - :func:`giql.canonical.decanonical_end`. + Input canonicalization is owned by ``CanonicalizeCoordinates`` (pass 2, + issue #122): every non-canonical interval-bearing operand is rewritten to + a canonical ``__giql_canon_*`` CTE before generation, so this emitter + consumes already-canonical 0-based half-open columns and applies no + in-emitter canonicalization arithmetic. The output round-trip back to the + target's declared encoding stays here — the ``disjoin_*`` columns and the + passed-through interval are synthesized at generation time and cannot be + reached by a pass-2 outermost-projection rewrite — driven by the original + encoding the pass preserves on the resolution. :param expression: GIQLDisjoin expression node :return: SQL string (a parenthesized WITH-CTE subquery) for the DISJOIN table """ - # Unpack the resolution metadata attached by ResolveOperatorRefs (pass 1). + # Unpack the resolution metadata attached by ResolveOperatorRefs (pass 1) + # and rewritten by CanonicalizeCoordinates (pass 2). target_ref, ref, ref_from = self._disjoin_resolution(expression) target_name = target_ref.name target_chrom, target_start, target_end = target_ref.cols - target_table = target_ref.table ref_chrom, ref_start, ref_end = ref.cols - ref_table = ref.table is_self_reference = ref.coverage_skippable - # Canonical target endpoints, qualified by the __giql_dj_tgt alias. + # The target's *declared* encoding, which disjoin_* output and the + # passed-through interval must round-trip back into. Pass 2 preserves it + # on the resolution when it wraps a non-canonical target (the slot's own + # Table is then None); a canonical target is left unwrapped and its slot + # Table carries the (identity) encoding. + output_table = self._disjoin_output_encoding(expression, target_ref) + + # Post-pass every operand is canonical 0-based half-open (a registered + # table is either identity-encoded or rewritten to a canonical CTE), so + # the physical columns are consumed verbatim with no canonicalization. t_chrom = f't."{target_chrom}"' - t_start = canonical_start(f't."{target_start}"', target_table) - t_end = canonical_end(f't."{target_end}"', target_table) - - # Canonical reference endpoints: unqualified for the breakpoint CTE, - # qualified by 'r' for the coverage EXISTS filter. - bp_start = canonical_start(f'"{ref_start}"', ref_table) - bp_end = canonical_end(f'"{ref_end}"', ref_table) - r_start = canonical_start(f'r."{ref_start}"', ref_table) - r_end = canonical_end(f'r."{ref_end}"', ref_table) - - # disjoin_start / disjoin_end are emitted in the target table's - # coordinate system so an output row carries one convention; the cut - # math above stays canonical internally. - out_start = decanonical_start("s.seg_start", target_table) - out_end = decanonical_end("s.seg_end", target_table) + t_start = f't."{target_start}"' + t_end = f't."{target_end}"' + + # Reference endpoints: unqualified for the breakpoint CTE, qualified by + # 'r' for the coverage EXISTS filter. + bp_start = f'"{ref_start}"' + bp_end = f'"{ref_end}"' + r_start = f'r."{ref_start}"' + r_end = f'r."{ref_end}"' + + # disjoin_start / disjoin_end are emitted in the target's declared + # coordinate system so an output row carries one convention; the cut math + # stays canonical internally. + out_start = decanonical_start("s.seg_start", output_table) + out_end = decanonical_end("s.seg_end", output_table) + passthrough = self._disjoin_passthrough(target_start, target_end, output_table) # Build the WITH clause one named fragment per __giql_dj_* CTE so each # block reads on its own. The `seg_end > seg_start` guard in the final @@ -361,7 +376,8 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: ) where_sql = " AND ".join(where_clauses) final_select = ( - f"SELECT t.*, s.kc AS disjoin_chrom, {out_start} AS disjoin_start, " + f"SELECT {passthrough}, s.kc AS disjoin_chrom, " + f"{out_start} AS disjoin_start, " f"{out_end} AS disjoin_end FROM __giql_dj_tgt AS t " f'JOIN __giql_dj_segs AS s ON t."{target_chrom}" = s.kc ' f'AND t."{target_start}" = s.ks AND t."{target_end}" = s.ke ' @@ -372,6 +388,64 @@ def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: f"{cuts_cte}, {segs_cte} {final_select})" ) + def _disjoin_output_encoding( + self, expression: GIQLDisjoin, target_ref: ResolvedRef + ) -> Table | None: + """Return the target's declared encoding for DISJOIN's output round-trip. + + ``CanonicalizeCoordinates`` (pass 2) records the original + :class:`~giql.table.Table` on the resolution when it wraps a non-canonical + target (blanking the slot's own ``table``). For an unwrapped target — a + canonical registered table, or any target when the pass did not run — the + slot's own ``table`` carries the (identity) encoding. + + :param expression: + GIQLDisjoin expression node + :param target_ref: + The resolved target reference (post pass 2) + :return: + The target's declared :class:`~giql.table.Table`, or ``None`` + """ + resolution = expression.meta.get(META_KEY) + if isinstance(resolution, OperatorResolution): + preserved = resolution.output_tables.get("this") + if preserved is not None: + return preserved + return target_ref.table + + def _disjoin_passthrough( + self, target_start: str, target_end: str, output_table: Table | None + ) -> str: + """Project the target's full row, de-canonicalizing the interval columns. + + When the target's declared encoding is canonical 0-based half-open the + row passes through as a plain ``t.*`` — the byte-identical identity fast + path. When it is non-canonical the interval columns, canonical inside + ``__giql_dj_tgt``, are de-canonicalized back into that encoding via a star + ``REPLACE`` so the passed-through interval matches the target's own + convention. (Only non-canonical targets are wrapped, so the ``REPLACE`` + appears only where a canonical CTE already shapes the SQL.) + + :param target_start: + Physical start column name + :param target_end: + Physical end column name + :param output_table: + The target's declared :class:`~giql.table.Table`, or ``None`` + :return: + The passthrough projection fragment (``t.*`` or a star ``REPLACE``) + """ + if output_table is None or ( + output_table.coordinate_system == "0based" + and output_table.interval_type == "half_open" + ): + return "t.*" + pt_start = decanonical_start(f't."{target_start}"', output_table) + pt_end = decanonical_end(f't."{target_end}"', output_table) + return ( + f't.* REPLACE ({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' + ) + def giqldistance_sql(self, expression: GIQLDistance) -> str: """Generate SQL CASE expression for DISTANCE function. diff --git a/src/giql/resolver.py b/src/giql/resolver.py index 2c6889a..9fa3104 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -346,12 +346,23 @@ class OperatorResolution: resolve (a literal range, or an unqualified column outside a current-table context) is left out and the generator raises its existing error. + output_tables : dict[str, Table] + Mapping from a slot's arg key to the *original* :class:`~giql.table.Table` + whose declared encoding that slot carried before + :func:`giql.canonicalizer.canonicalize_coordinates` (pass 2) wrapped it + in a canonical CTE and blanked its ``ResolvedRef.table``. The pass + populates this for every slot it wraps so the operator's emitter can + round-trip *synthesized* output columns (DISJOIN's ``disjoin_*`` and its + passed-through interval) back into that encoding — columns that do not + exist as AST in pass 2 and that a ``SELECT *`` consumer hides from any + outer-projection rewrite. Empty until pass 2 wraps a slot. """ operator: str slots: dict[str, ResolvedRef | ResolvedInterval] deferrals: dict[str, SlotDeferral] = field(default_factory=dict) columns: dict[str, ResolvedColumn] = field(default_factory=dict) + output_tables: dict[str, Table] = field(default_factory=dict) def slot(self, arg: str) -> ResolvedRef | ResolvedInterval | None: """Return the resolved metadata for slot *arg*, or ``None``.""" From d90bdac3c9952a961b80c5fbe603654e4181a93b Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 15:08:44 -0400 Subject: [PATCH 046/142] test: Cover DISJOIN canonicalization and relocate moved assertions Pin that a non-canonical DISJOIN target or reference produces the canonical wrapper CTE with no inline arithmetic, that a canonical target gets no wrapper and a plain passthrough, that the omitted self-reference dedupes target and reference to one wrapper, and that the passed-through interval round-trips to the target encoding. Move the canonicalizer projection and DISJOIN encoding assertions to match the arithmetic's new home in the wrapper CTE, and save and restore the GIQL_CANONICALIZE flag in the gating tests now that it is a permanent class attribute. --- tests/test_canonicalizer.py | 57 +++++++---- tests/test_disjoin_transpilation.py | 153 ++++++++++++++++++++++++++-- 2 files changed, 181 insertions(+), 29 deletions(-) diff --git a/tests/test_canonicalizer.py b/tests/test_canonicalizer.py index efffb0f..6899e9a 100644 --- a/tests/test_canonicalizer.py +++ b/tests/test_canonicalizer.py @@ -108,18 +108,20 @@ def test_canonical_table_sql_unchanged(self): assert actual == expected assert CANON_PREFIX not in actual - def test_non_canonical_table_sql_unchanged(self): + def test_non_canonical_table_sql_unchanged(self, monkeypatch): """Test that a non-canonical table's SQL is byte-identical through the pass. Given: - A DISJOIN over a non-canonical (1based/closed) registered table and no - operator opted into canonicalization. + A DISJOIN over a non-canonical (1based/closed) registered table with + its GIQL_CANONICALIZE flag explicitly toggled off (the gating an + unmigrated operator relies on). When: Transpiling the query and comparing against a pass-bypassed run. Then: No wrapper CTE is synthesized and the SQL is unchanged. """ # Arrange + monkeypatch.setattr(GIQLDisjoin, "GIQL_CANONICALIZE", False, raising=False) query = "SELECT * FROM DISJOIN(variants)" variants = Table("variants", coordinate_system="1based", interval_type="closed") tables = Tables() @@ -134,17 +136,19 @@ def test_non_canonical_table_sql_unchanged(self): assert actual == expected assert CANON_PREFIX not in actual - def test_pass_returns_expression_with_no_with_added(self): + def test_pass_returns_expression_with_no_with_added(self, monkeypatch): """Test that the no-op pass adds no WITH clause to the AST. Given: - A non-canonical DISJOIN AST and no operator opted in. + A non-canonical DISJOIN AST with its GIQL_CANONICALIZE flag explicitly + toggled off. When: Running canonicalize_coordinates directly. Then: No canonical CTE is present on the returned tree. """ # Arrange + monkeypatch.setattr(GIQLDisjoin, "GIQL_CANONICALIZE", False, raising=False) tables = _tables(("1based", "closed")) # Act @@ -175,8 +179,8 @@ def test_zero_based_closed_projection(self, disjoin_opted_in): # Assert body = _canon_ctes(ast)[0].this.sql() - assert "start AS start" in body - assert "(end + 1) AS end" in body + assert '"start" AS "start"' in body + assert '("end" + 1) AS "end"' in body def test_one_based_half_open_projection(self, disjoin_opted_in): """Test the canonical projection for a 1based/half_open table. @@ -196,8 +200,8 @@ def test_one_based_half_open_projection(self, disjoin_opted_in): # Assert body = _canon_ctes(ast)[0].this.sql() - assert "(start - 1) AS start" in body - assert "(end - 1) AS end" in body + assert '("start" - 1) AS "start"' in body + assert '("end" - 1) AS "end"' in body def test_one_based_closed_projection(self, disjoin_opted_in): """Test the canonical projection for a 1based/closed table. @@ -217,8 +221,8 @@ def test_one_based_closed_projection(self, disjoin_opted_in): # Assert body = _canon_ctes(ast)[0].this.sql() - assert "(start - 1) AS start" in body - assert "end AS end" in body + assert '("start" - 1) AS "start"' in body + assert '"end" AS "end"' in body def test_projection_preserves_custom_column_names(self, disjoin_opted_in): """Test the projection uses a table's physical column names. @@ -228,7 +232,8 @@ def test_projection_preserves_custom_column_names(self, disjoin_opted_in): When: Running pass 2. Then: - The wrapper exposes the canonical interval under those same names. + The wrapper exposes the canonical interval under those same names; + chrom (never offset) flows through the star untouched. """ # Arrange variants = Table( @@ -247,9 +252,9 @@ def test_projection_preserves_custom_column_names(self, disjoin_opted_in): # Assert body = _canon_ctes(ast)[0].this.sql() - assert "chr AS chr" in body - assert "(lo - 1) AS lo" in body - assert "hi AS hi" in body + assert body.startswith("SELECT *") + assert '("lo" - 1) AS "lo"' in body + assert '"hi" AS "hi"' in body class TestPassThrough: @@ -548,8 +553,10 @@ def test_wrap_iff_non_canonical(self, coordinate_system, interval_type): wrapped slot always becomes a Table-free CTE ref. """ # Arrange - # A plain try/finally toggles the class flag: @given re-runs the body - # many times under one function-scoped fixture, so monkeypatch is unsafe. + # DISJOIN is opted into canonicalization by default (issue #122); a plain + # save/restore toggle pins it on, since @given re-runs the body many times + # under one function-scoped fixture, making monkeypatch unsafe. + previous = GIQLDisjoin.GIQL_CANONICALIZE GIQLDisjoin.GIQL_CANONICALIZE = True tables = _tables((coordinate_system, interval_type)) is_canonical = coordinate_system == "0based" and interval_type == "half_open" @@ -558,7 +565,7 @@ def test_wrap_iff_non_canonical(self, coordinate_system, interval_type): try: ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) finally: - del GIQLDisjoin.GIQL_CANONICALIZE + GIQLDisjoin.GIQL_CANONICALIZE = previous # Assert ctes = _canon_ctes(ast) @@ -580,18 +587,26 @@ def test_flags_off_is_always_noop(self, coordinate_system, interval_type): """Test that the pass never mutates the tree while flags are off. Given: - A DISJOIN over a registered table with any encoding and no operator - opted in. + A DISJOIN over a registered table with any encoding and its + GIQL_CANONICALIZE flag explicitly toggled off (the gating an + unmigrated operator relies on). When: Running passes 1 and 2. Then: No canonical CTE is ever synthesized. """ # Arrange + # Save/restore the class flag off: @given re-runs the body many times + # under one function-scoped fixture, making monkeypatch unsafe. + previous = GIQLDisjoin.GIQL_CANONICALIZE + GIQLDisjoin.GIQL_CANONICALIZE = False tables = _tables((coordinate_system, interval_type)) # Act - ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + try: + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + finally: + GIQLDisjoin.GIQL_CANONICALIZE = previous # Assert assert _canon_ctes(ast) == [] diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index 16c2bc3..34412b9 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -115,7 +115,9 @@ def test_giqldisjoin_sql_should_canonicalize_one_based_closed_target(self): When: Transpiling a DISJOIN query. Then: - It should shift the start to canonical 0-based coordinates. + It should shift the start to canonical 0-based coordinates in a + synthesized __giql_canon_ wrapper CTE, with no inline shift left in + the emitter's cut arithmetic. """ # Arrange & act sql = transpile( @@ -130,7 +132,11 @@ def test_giqldisjoin_sql_should_canonicalize_one_based_closed_target(self): ) # Assert - assert '(t."start" - 1)' in sql + # Canonicalization now lives in the CanonicalizeCoordinates wrapper CTE, + # not as inline emitter arithmetic. + assert "__giql_canon_" in sql + assert '("start" - 1) AS "start"' in sql + assert '(t."start" - 1)' not in sql def test_giqldisjoin_sql_should_emit_disjoin_start_in_target_encoding(self): """Test that disjoin_start is emitted in the target's encoding. @@ -403,8 +409,8 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_one_based_cl When: Transpiling to SQL. Then: - It should omit the coverage EXISTS subquery and still emit the - canonical 0-based shift on the target endpoints. + It should omit the coverage EXISTS subquery and still apply the + canonical 0-based shift on the target endpoints in the wrapper CTE. """ # Arrange & act sql = transpile( @@ -420,7 +426,7 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_one_based_cl # Assert assert "EXISTS (" not in sql - assert '(t."start" - 1)' in sql + assert '("start" - 1) AS "start"' in sql def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_custom_column_names( self, @@ -460,8 +466,9 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_explicit_self_reference_ When: Transpiling to SQL. Then: - It should omit the coverage EXISTS subquery and still emit the - canonical 0-based shift on the breakpoint CTE endpoints. + It should omit the coverage EXISTS subquery and still apply the + canonical 0-based shift in the wrapper CTE shared by target and + (self-)reference. """ # Arrange & act sql = transpile( @@ -477,7 +484,11 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_explicit_self_reference_ # Assert assert "EXISTS (" not in sql - assert '("start" - 1)' in sql + assert '("start" - 1) AS "start"' in sql + # Target and explicit self-reference dedupe to a single wrapper CTE: one + # CTE definition plus the two FROM references (target + reference). + assert sql.count("__giql_canon_") == 3 + assert "SELECT * REPLACE" in sql def test_giqldisjoin_sql_should_emit_one_exists_clause_when_query_mixes_self_and_distinct_reference_disjoins( self, @@ -695,3 +706,129 @@ def test_giqldisjoin_sql_should_not_see_cte_declared_in_sibling_derived_table( "(WITH refs AS (SELECT 1) SELECT * FROM refs) AS sub", tables=["features"], ) + + +class TestDisjoinCanonicalization: + """DISJOIN consumes CanonicalizeCoordinates (pass 2) output (issue #122).""" + + def test_giqldisjoin_sql_should_wrap_non_canonical_target_in_canon_cte(self): + """Test that a non-canonical target is wrapped by the canonicalize pass. + + Given: + A self-mode DISJOIN over a 1-based closed target. + When: + Transpiling to SQL. + Then: + It should synthesize a __giql_canon_ wrapper CTE doing the canonical + shift via a star REPLACE, with no canonicalization arithmetic left + inline in the emitter's cut/breakpoint expressions. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features)", + tables=[ + Table("features", coordinate_system="1based", interval_type="closed") + ], + ) + + # Assert + assert "__giql_canon_" in sql + assert "SELECT * REPLACE" in sql + assert '("start" - 1) AS "start"' in sql + # No inline canonicalization arithmetic survives in the emitter output. + assert '(t."start" - 1)' not in sql + assert '("start" - 1) AS pos' not in sql + + def test_giqldisjoin_sql_should_wrap_non_canonical_reference_in_canon_cte(self): + """Test that a non-canonical explicit reference is wrapped by the pass. + + Given: + A two-table DISJOIN whose distinct reference table is 1-based closed + and whose target is canonical. + When: + Transpiling to SQL. + Then: + It should wrap the reference in a __giql_canon_ CTE and emit no inline + shift on the breakpoint endpoints. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features, reference := refs)", + tables=[ + "features", + Table("refs", coordinate_system="1based", interval_type="closed"), + ], + ) + + # Assert + assert "__giql_canon_" in sql + assert "SELECT * REPLACE" in sql + # The breakpoint CTE reads the canonical endpoints verbatim, no inline shift. + assert '("start" - 1) AS pos' not in sql + + def test_giqldisjoin_sql_should_not_wrap_canonical_target(self): + """Test that a canonical target produces no wrapper CTE (identity fast path). + + Given: + A self-mode DISJOIN over a canonical 0-based half-open target. + When: + Transpiling to SQL. + Then: + It should synthesize no __giql_canon_ wrapper CTE and pass the row + through as a plain ``t.*`` with no star REPLACE. + """ + # Arrange & act + sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) + + # Assert + assert "__giql_canon_" not in sql + assert "REPLACE" not in sql + assert "t.*," in sql + + def test_giqldisjoin_sql_should_dedupe_self_reference_to_one_canon_cte(self): + """Test that an omitted self-reference shares one wrapper with the target. + + Given: + A self-mode (omitted reference) DISJOIN over a 1-based closed target, + where target and defaulted reference resolve to the same relation. + When: + Transpiling to SQL. + Then: + It should synthesize exactly one canonical wrapper CTE, referenced by + both the target and the reference CTEs (one definition plus two FROM + references). + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features)", + tables=[ + Table("features", coordinate_system="1based", interval_type="closed") + ], + ) + + # Assert + assert sql.count("AS (SELECT * REPLACE") == 1 + assert sql.count("__giql_canon_") == 3 + + def test_giqldisjoin_sql_should_emit_passthrough_interval_in_target_encoding(self): + """Test that the passed-through interval is de-canonicalized to the target. + + Given: + A self-mode DISJOIN over a 1-based closed target whose row passes + through canonical inside the wrapper CTE. + When: + Transpiling to SQL. + Then: + It should de-canonicalize the passed-through start/end back into the + target's encoding via a star REPLACE on the final projection. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features)", + tables=[ + Table("features", coordinate_system="1based", interval_type="closed") + ], + ) + + # Assert + assert 't.* REPLACE ((t."start" + 1) AS "start", t."end" AS "end")' in sql From c99107b781fbf8ab887d4c8d9c053f6cd3f50ba0 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 15:08:44 -0400 Subject: [PATCH 047/142] docs: Note the non-canonical-encoding engine requirement Document that a table declaring a non-canonical encoding canonicalizes through a SELECT * REPLACE wrapper, which DuckDB, BigQuery, Snowflake, and ClickHouse support but PostgreSQL, SQLite, and DataFusion do not. Note the default 0-based half-open path is unaffected, give the store-canonical and convert-in-a-CTE workarounds, and link the tracked follow-up for dialect-aware emission. --- docs/transpilation/schema-mapping.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/transpilation/schema-mapping.rst b/docs/transpilation/schema-mapping.rst index 453c289..2cba2ea 100644 --- a/docs/transpilation/schema-mapping.rst +++ b/docs/transpilation/schema-mapping.rst @@ -173,6 +173,23 @@ If your data uses 1-based coordinates (like VCF or GFF), configure the ], ) +.. note:: + + **Non-canonical encodings currently require a DuckDB-compatible engine.** + When a table declares an encoding other than the default 0-based half-open + (for example ``coordinate_system="1based"`` or ``interval_type="closed"``), + GIQL canonicalizes its coordinates by wrapping the relation in a hidden CTE + that uses ``SELECT * REPLACE (...)`` syntax. That syntax is supported by + DuckDB, BigQuery, Snowflake, and ClickHouse, but **not** by PostgreSQL, + SQLite, or DataFusion. Tables in the default 0-based half-open encoding are + unaffected -- they take an identity fast path that emits portable SQL. + + To target a non-``REPLACE`` engine today, store your data in 0-based + half-open form, or convert it explicitly in a CTE and reference that CTE + (which GIQL treats as already canonical). Making canonicalization emit + portable SQL on every engine is tracked in + `#132 `_. + Working with Point Features ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From d0b35c12ab2991f9b78a8c8b870b3c6ec894a7f2 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 16:59:21 -0400 Subject: [PATCH 048/142] refactor: Move NEAREST, DISTANCE, and predicate canonicalization into pass 2 Opt the remaining interval operators into the CanonicalizeCoordinates pass and remove the inline coordinate arithmetic from their emitters. The generator no longer imports or calls canonical_start and canonical_end at all; pass 2 now owns every input-side canonicalization. The migration splits by operand ownership. NEAREST's target is a relation the operator owns, so it is wrapped in a canonical CTE exactly as DISJOIN is, with the target row round-tripped back to the declared encoding through the recorded output encoding. DISTANCE operands, predicate operands, and NEAREST's column or implicit-outer reference are column operands that share the enclosing query's FROM relation with the user's own projection; wrapping that relation would canonicalize columns the user selects directly, so the pass instead rewrites those operands' resolved start and end fragments in place and blanks their table. The emitter consumes the resolved fragments verbatim. DISTANCE returns an encoding-invariant base count and the predicates return booleans, so neither needs output de-canonicalization; only NEAREST's passed-through row does. --- src/giql/canonicalizer.py | 137 +++++++++++++++++++++++++++- src/giql/expressions.py | 32 +++++++ src/giql/generators/base.py | 175 +++++++++++++++++++++++++++--------- 3 files changed, 299 insertions(+), 45 deletions(-) diff --git a/src/giql/canonicalizer.py b/src/giql/canonicalizer.py index 714a007..78e1948 100644 --- a/src/giql/canonicalizer.py +++ b/src/giql/canonicalizer.py @@ -65,14 +65,40 @@ A migrated operator's *output* columns must land back in the target relation's declared encoding. Epic #114 step 6 envisioned a rewrite of the outermost ``SELECT`` projection, but that placement is wrong for a table function: DISJOIN -synthesizes its ``disjoin_*`` output and its passed-through interval at +and NEAREST synthesize their output columns and pass whole source rows through at *generation* time, so those columns do not exist as AST in this pass, and a ``SELECT *`` consumer hides them from any outer-projection rewrite. So :func:`_decanonicalize_outputs` instead records each wrapped slot's *original* :class:`~giql.table.Table` on the operator's :class:`~giql.resolver.OperatorResolution`, and the operator's emitter reads it to de-canonicalize those synthesized columns where it generates them (DISJOIN, -issue #122). +issue #122; NEAREST's target row passthrough, issue #123). + +Column / interval operands (epic #114, step 8 / issue #123) +----------------------------------------------------------- +A reference slot (DISJOIN/NEAREST target, DISJOIN reference) *owns* the relation +it names, so the pass can wrap that whole relation in a ``__giql_canon_*`` CTE and +redirect the slot's AST node to it. A *column* operand cannot be wrapped that way: +``DISTANCE(a.interval, b.interval)`` and ``a.interval INTERSECTS b.interval`` +reference an alias bound in the *enclosing* query's ``FROM`` / ``JOIN``, shared +with the user's own projection (``SELECT a.start, DISTANCE(...)``). Rewriting that +``FROM`` source to a canonical CTE would silently canonicalize the user's own +``a.start`` too — a behavior change. NEAREST's column / implicit-outer +``reference`` slot is the same shape (an alias from the outer LATERAL relation). + +For those operands the pass therefore canonicalizes the resolution metadata *in +place* rather than synthesizing a CTE: :func:`_canonicalize_column_operands` +rewrites each :class:`~giql.resolver.ResolvedColumn` (DISTANCE's two operands and +the spatial predicates' column operands) and each non-table +:class:`~giql.resolver.ResolvedInterval` (NEAREST's ``column`` / +``implicit_outer`` reference) so its ``start`` / ``end`` fragments carry the +canonical 0-based half-open arithmetic and its ``table`` is blanked. The emitter +then consumes the fragments verbatim — no in-emitter +:func:`giql.canonical.canonical_start` / ``canonical_end``. The arithmetic is the +same the emitter used to emit inline, so the SQL stays byte-identical for these +operands; only the *owner* of the arithmetic moves from the generator to the pass. +A ``literal_range`` interval is already canonical and is left untouched, and an +operand whose ``table`` is already canonical (or ``None``) is a no-op. """ from __future__ import annotations @@ -94,6 +120,8 @@ from giql.expressions import Within from giql.resolver import META_KEY from giql.resolver import OperatorResolution +from giql.resolver import ResolvedColumn +from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef from giql.table import Table @@ -146,9 +174,16 @@ def canonicalize_coordinates(expression: exp.Expression) -> exp.Expression: The same *expression*, with canonical wrapper CTEs inserted and migrated operator slots rewritten (none, while every flag is off). """ + # Column / interval operands (DISTANCE, predicates, NEAREST's non-table + # reference) canonicalize their metadata in place; this is independent of the + # ref-slot CTE synthesis below and runs for every opted-in operator. + _canonicalize_column_operands(expression) + targets = _collect_targets(expression) if not targets: - # No opted-in operator carries a non-canonical operand: strict no-op. + # No opted-in operator carries a non-canonical *reference-slot* operand: + # no wrapper CTE is synthesized (the in-place column canonicalization + # above already ran). return expression taken = _collect_taken_names(expression) @@ -223,6 +258,102 @@ def _is_canonical(table: Table | None) -> bool: return table.coordinate_system == "0based" and table.interval_type == "half_open" +def _canonicalize_column_operands(expression: exp.Expression) -> None: + """Canonicalize column / interval operand metadata in place for opted-in ops. + + For every opted-in operator (``GIQL_CANONICALIZE``) carrying column operands + (DISTANCE's two operands and the spatial predicates' column operands, in the + :attr:`OperatorResolution.columns` channel) or a non-table interval reference + (NEAREST's ``column`` / ``implicit_outer`` slot, in + :attr:`OperatorResolution.slots`), rewrite the operand's ``start`` / ``end`` + SQL fragments to carry the canonical 0-based half-open arithmetic for its + declared encoding and blank its ``table``. + + This replaces the in-emitter :func:`giql.canonical.canonical_start` / + ``canonical_end`` wrapping for those operands (epic #114, step 8). Unlike the + ref-slot CTE synthesis, no relation is wrapped: the operand references an + alias bound in the enclosing query's ``FROM`` shared with the user's own + projection, so canonicalizing the whole relation would change unrelated + columns. Operands already canonical (``table`` is ``None`` or 0-based + half-open) are left untouched, keeping their SQL byte-identical; a + ``literal_range`` interval is already canonical and is skipped. + """ + for node in expression.walk(): + if not isinstance(node, _OPERATORS): + continue + if not getattr(node, "GIQL_CANONICALIZE", False): + continue + resolution = node.meta.get(META_KEY) + if not isinstance(resolution, OperatorResolution): + continue + for arg, column in list(resolution.columns.items()): + resolution.columns[arg] = _canonicalize_column(column) + for arg, slot in list(resolution.slots.items()): + if isinstance(slot, ResolvedInterval): + resolution.slots[arg] = _canonicalize_interval(slot) + + +def _canonicalize_column(column: ResolvedColumn) -> ResolvedColumn: + """Return *column* with canonical start/end fragments and a blanked table. + + A no-op (returns *column* unchanged) when its backing table is already + canonical or ``None``. + """ + if _is_canonical(column.table): + return column + return replace( + column, + start=_canonical_start_sql(column.start, column.table), + end=_canonical_end_sql(column.end, column.table), + table=None, + ) + + +def _canonicalize_interval(interval: ResolvedInterval) -> ResolvedInterval: + """Return *interval* with canonical start/end fragments and a blanked table. + + A no-op for a ``literal_range`` (already canonical, no table) and for any + interval whose backing table is already canonical or ``None``. + """ + if interval.kind == "literal_range" or _is_canonical(interval.table): + return interval + return replace( + interval, + start=_canonical_start_sql(interval.start, interval.table), + end=_canonical_end_sql(interval.end, interval.table), + table=None, + ) + + +def _canonical_start_sql(start: str, table: Table | None) -> str: + """SQL-fragment analog of :func:`giql.canonical.canonical_start`. + + - ``0based``: ``start`` (identity) + - ``1based``: ``(start - 1)`` + """ + if table is None or table.coordinate_system == "0based": + return start + return f"({start} - 1)" + + +def _canonical_end_sql(end: str, table: Table | None) -> str: + """SQL-fragment analog of :func:`giql.canonical.canonical_end`. + + - ``0based`` / ``half_open``: ``end`` (identity) + - ``0based`` / ``closed``: ``(end + 1)`` + - ``1based`` / ``half_open``: ``(end - 1)`` + - ``1based`` / ``closed``: ``end`` (identity) + """ + if table is None: + return end + key = (table.coordinate_system, table.interval_type) + if key == ("0based", "closed"): + return f"({end} + 1)" + if key == ("1based", "half_open"): + return f"({end} - 1)" + return end + + def _collect_taken_names(expression: exp.Expression) -> set[str]: """Collect every relation name already in use across all scopes. diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 52e5fce..e8de483 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -99,12 +99,26 @@ class SpatialPredicate(exp.Binary): pass +#: Opt the spatial predicates, DISTANCE, and NEAREST into the +#: CanonicalizeCoordinates pass (epic #114 step 8, issue #123). With this flag +#: set, pass 2 canonicalizes each operator's interval operands: a non-table column +#: operand (DISTANCE / predicate operand, NEAREST's column / implicit-outer +#: reference) has its resolution metadata rewritten in place to canonical 0-based +#: half-open arithmetic, and a registered-table reference slot (NEAREST's target) +#: is wrapped in a ``__giql_canon_*`` CTE. The emitter then consumes already- +#: canonical fragments with no in-emitter canonicalization. Identity (0-based +#: half-open) operands are left untouched and the emitted SQL stays byte-identical. +_CANONICALIZE = True + + class Intersects(SpatialPredicate): """INTERSECTS spatial predicate. Example: column INTERSECTS 'chr1:1000-2000' """ + GIQL_CANONICALIZE = _CANONICALIZE + GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), SlotSpec("expression", frozenset({"literal_range", "column"}), required=True), @@ -117,6 +131,8 @@ class Contains(SpatialPredicate): Example: column CONTAINS 'chr1:1500' """ + GIQL_CANONICALIZE = _CANONICALIZE + GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), SlotSpec("expression", frozenset({"literal_range", "column"}), required=True), @@ -129,6 +145,8 @@ class Within(SpatialPredicate): Example: column WITHIN 'chr1:1000-5000' """ + GIQL_CANONICALIZE = _CANONICALIZE + GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), SlotSpec("expression", frozenset({"literal_range", "column"}), required=True), @@ -150,6 +168,8 @@ class SpatialSetPredicate(exp.Expression): "ranges": True, } + GIQL_CANONICALIZE = _CANONICALIZE + GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), SlotSpec( @@ -259,6 +279,8 @@ class GIQLDistance(exp.Func): "signed": False, # Optional: boolean for directional distance } + GIQL_CANONICALIZE = _CANONICALIZE + GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), SlotSpec("expression", frozenset({"column"}), required=True), @@ -296,6 +318,16 @@ class GIQLNearest(exp.Func): "signed": False, # Optional: directional distance } + #: Opt NEAREST into the CanonicalizeCoordinates pass (epic #114 step 8, issue + #: #123). Pass 2 wraps a non-canonical *target* (the ``this`` registered-table + #: ref slot) in a ``__giql_canon_*`` CTE — so the emitter reads canonical + #: target columns and de-canonicalizes its ``*`` row passthrough back to the + #: declared encoding — and canonicalizes a non-table ``column`` / + #: ``implicit_outer`` reference's metadata in place. Identity (0-based + #: half-open) operands are left untouched and the emitted SQL stays + #: byte-identical. + GIQL_CANONICALIZE = _CANONICALIZE + GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), SlotSpec( diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 2bbf918..1874094 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -1,8 +1,6 @@ from sqlglot import exp from sqlglot.generator import Generator -from giql.canonical import canonical_end -from giql.canonical import canonical_start from giql.canonical import decanonical_end from giql.canonical import decanonical_start from giql.constants import DEFAULT_CHROM_COL @@ -159,21 +157,29 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: ) table_name = target_ref.name target_chrom, target_start, target_end = target_ref.cols - target_table = target_ref.table + + # The target's *declared* encoding, which the passed-through target row + # (SELECT {table_name}.*) must round-trip back into. CanonicalizeCoordinates + # (pass 2) preserves it on the resolution when it wraps a non-canonical + # target in a __giql_canon_* CTE (the slot's own Table is then None); a + # canonical target is left unwrapped and its slot Table carries the + # (identity) encoding. The synthesized `distance` column is encoding- + # invariant (a count of bases) and needs no round-trip. + output_table = self._nearest_output_encoding(expression, target_ref) + passthrough = self._nearest_passthrough( + table_name, target_start, target_end, output_table + ) # Reference interval (a ResolvedInterval from the pass). An unresolved - # reference re-raises the generator's historical diagnostic. + # reference re-raises the generator's historical diagnostic. Input + # canonicalization is owned by CanonicalizeCoordinates (pass 2, issue + # #123): a literal range is already canonical, and a column / implicit- + # outer reference's endpoints are canonicalized in place by the pass, so + # the emitter consumes the fragments verbatim with no canonicalization. ref = resolution.slot("reference") if not isinstance(ref, ResolvedInterval): self._raise_nearest_reference_error(expression, mode, resolution) - if ref.kind == "literal_range": - # Literal endpoints are already canonical 0-based half-open. - ref_chrom, ref_start, ref_end = ref.chrom, ref.start, ref.end - else: - # Column / implicit-outer endpoints are raw; canonicalize here. - ref_chrom = ref.chrom - ref_start = canonical_start(ref.start, ref.table) - ref_end = canonical_end(ref.end, ref.table) + ref_chrom, ref_start, ref_end = ref.chrom, ref.start, ref.end # Extract parameters k = expression.args.get("k") @@ -193,14 +199,23 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: target_strand = None if is_stranded: ref_strand = ref.strand - if target_table and target_table.strand_col: - target_strand = f'{table_name}."{target_table.strand_col}"' - - # Distance math below assumes 0-based half-open. - target_start_expr = canonical_start( - f'{table_name}."{target_start}"', target_table - ) - target_end_expr = canonical_end(f'{table_name}."{target_end}"', target_table) + # When pass 2 wraps a non-canonical target its slot Table is blanked, + # so the strand column name comes from the *declared* encoding the + # pass preserved (output_table). The canon CTE's SELECT * REPLACE + # passes the strand column through unchanged under its physical name, + # so the qualifier stays the relation NEAREST selects from. + if output_table and output_table.strand_col: + target_strand = f'{table_name}."{output_table.strand_col}"' + + # Distance math below assumes 0-based half-open. Input canonicalization + # is owned by CanonicalizeCoordinates (pass 2, issue #123): a + # non-canonical target is rewritten to a canonical __giql_canon_* CTE + # before generation (table_name then names the CTE), so the target + # endpoints are consumed verbatim with no in-emitter canonicalization. The + # output round-trip of the passed-through target row stays here (see the + # SELECT projection below). + target_start_expr = f'{table_name}."{target_start}"' + target_end_expr = f'{table_name}."{target_end}"' # Build distance calculation using CASE expression # For NEAREST: ORDER BY absolute distance, but RETURN signed distance @@ -239,7 +254,7 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: # Standalone mode: direct ORDER BY + LIMIT # Return signed distance, but order by absolute distance sql = f"""( - SELECT {table_name}.*, {distance_expr} AS distance + SELECT {passthrough}, {distance_expr} AS distance FROM {table_name} WHERE {where_sql} ORDER BY {abs_distance_expr} @@ -261,7 +276,7 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: # LATERAL mode: subquery for k-nearest neighbors # Return signed distance, but order by absolute distance sql = f"""( - SELECT {table_name}.*, {distance_expr} AS distance + SELECT {passthrough}, {distance_expr} AS distance FROM {table_name} WHERE {where_sql} ORDER BY {abs_distance_expr} @@ -270,6 +285,77 @@ def giqlnearest_sql(self, expression: GIQLNearest) -> str: return sql.strip() + def _nearest_output_encoding( + self, expression: GIQLNearest, target_ref: ResolvedRef + ) -> Table | None: + """Return the target's declared encoding for NEAREST's row passthrough. + + ``CanonicalizeCoordinates`` (pass 2) records the original + :class:`~giql.table.Table` on the resolution when it wraps a non-canonical + target in a ``__giql_canon_*`` CTE (blanking the slot's own ``table``). + For an unwrapped target — a canonical registered table, or any target when + the pass did not run — the slot's own ``table`` carries the (identity) + encoding. + + :param expression: + GIQLNearest expression node + :param target_ref: + The resolved target reference (post pass 2) + :return: + The target's declared :class:`~giql.table.Table`, or ``None`` + """ + resolution = expression.meta.get(META_KEY) + if isinstance(resolution, OperatorResolution): + preserved = resolution.output_tables.get("this") + if preserved is not None: + return preserved + return target_ref.table + + def _nearest_passthrough( + self, + table_name: str, + target_start: str, + target_end: str, + output_table: Table | None, + ) -> str: + """Project the target's full row, de-canonicalizing the interval columns. + + NEAREST passes the whole target row through (``SELECT {table_name}.*``) + alongside the synthesized, encoding-invariant ``distance`` column. When the + target's declared encoding is canonical 0-based half-open the row passes + through as a plain ``{table_name}.*`` — the byte-identical identity fast + path. When it is non-canonical the interval columns, canonical inside the + ``__giql_canon_*`` CTE the target was rewritten to, are de-canonicalized + back into that encoding via a star ``REPLACE`` so the passed-through + interval matches the target's own convention. (Only non-canonical targets + are wrapped, so the ``REPLACE`` appears only where a canonical CTE already + shapes the SQL.) + + :param table_name: + The relation the row is selected from (the canon CTE name when wrapped, + else the registered table name) — also the column qualifier. + :param target_start: + Physical start column name + :param target_end: + Physical end column name + :param output_table: + The target's declared :class:`~giql.table.Table`, or ``None`` + :return: + The passthrough projection fragment (``{table_name}.*`` or a star + ``REPLACE``) + """ + if output_table is None or ( + output_table.coordinate_system == "0based" + and output_table.interval_type == "half_open" + ): + return f"{table_name}.*" + pt_start = decanonical_start(f'{table_name}."{target_start}"', output_table) + pt_end = decanonical_end(f'{table_name}."{target_end}"', output_table) + return ( + f"{table_name}.* REPLACE " + f'({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' + ) + def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: """Generate SQL for the DISJOIN table function. @@ -475,21 +561,22 @@ def giqldistance_sql(self, expression: GIQLDistance) -> str: strand_a = col_a.strand if stranded else None strand_b = col_b.strand if stranded else None - # Distance math below assumes 0-based half-open. - start_a = canonical_start(col_a.start, col_a.table) - end_a = canonical_end(col_a.end, col_a.table) - start_b = canonical_start(col_b.start, col_b.table) - end_b = canonical_end(col_b.end, col_b.table) + # Distance math below assumes 0-based half-open. Input canonicalization is + # owned by CanonicalizeCoordinates (pass 2, issue #123): each operand's + # start/end fragments are canonicalized in place by the pass, so the + # emitter consumes them verbatim with no in-emitter canonicalization. The + # returned distance is an encoding-invariant base count, so it needs no + # output de-canonicalization. # Generate CASE expression return self._generate_distance_case( col_a.chrom, - start_a, - end_a, + col_a.start, + col_a.end, strand_a, col_b.chrom, - start_b, - end_b, + col_b.start, + col_b.end, strand_b, stranded=stranded, signed=signed, @@ -715,12 +802,14 @@ def _generate_range_predicate( :return: SQL predicate string """ - # Canonicalize the raw physical endpoints to 0-based half-open. The - # alias-qualified column fragments come pre-resolved on the - # ResolvedColumn; canonicalization stays here (epic #114 step #123). + # The alias-qualified column fragments come pre-resolved on the + # ResolvedColumn, already canonicalized to 0-based half-open by + # CanonicalizeCoordinates (pass 2, issue #123). The predicate returns a + # boolean, which is encoding-invariant, so no output de-canonicalization + # is needed. chrom_col = column.chrom - start_col = canonical_start(column.start, column.table) - end_col = canonical_end(column.end, column.table) + start_col = column.start + end_col = column.end chrom = parsed_range.chromosome start = parsed_range.start @@ -774,14 +863,16 @@ def _generate_column_join( :return: SQL predicate string """ - # Canonicalize each side's raw physical endpoints; the alias-qualified - # chrom/start/end fragments come pre-resolved on the ResolvedColumns. + # The alias-qualified chrom/start/end fragments come pre-resolved on the + # ResolvedColumns, already canonicalized to 0-based half-open by + # CanonicalizeCoordinates (pass 2, issue #123). The predicate returns a + # boolean (encoding-invariant), so no output de-canonicalization is needed. l_chrom = left.chrom r_chrom = right.chrom - l_start = canonical_start(left.start, left.table) - l_end = canonical_end(left.end, left.table) - r_start = canonical_start(right.start, right.table) - r_end = canonical_end(right.end, right.table) + l_start = left.start + l_end = left.end + r_start = right.start + r_end = right.end if op_type == "intersects": # Ranges overlap if: chrom1 = chrom2 AND start1 < end2 AND end1 > start2 From 5129fba476de6f54cd6bb2bd24890b5699d28bf3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 16:59:22 -0400 Subject: [PATCH 049/142] test: Cover remaining-operator canonicalization and relocate assertions Pin that a non-canonical DISTANCE or predicate column operand canonicalizes in place with no wrapper CTE, that a non-canonical NEAREST target is wrapped and its row round-trips to the declared encoding, and that canonical operands stay byte-identical. Relocate the generator unit tests whose arithmetic moved from the emitter into the pass to drive through the full transpile pipeline (or an explicit two-pass helper for the binned-join INTERSECTS path), preserving each assertion's intent. Flag toggles save and restore the GIQL_CANONICALIZE attribute. --- tests/generators/test_base.py | 291 +++++++++++++++++++++------------- tests/test_canonicalizer.py | 193 ++++++++++++++++++++++ 2 files changed, 370 insertions(+), 114 deletions(-) diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index 9cd6598..d316157 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -12,12 +12,33 @@ from sqlglot import parse_one from giql import Table +from giql import transpile +from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expressions import GIQLNearest from giql.generators import BaseGIQLGenerator +from giql.resolver import resolve_operator_refs from giql.table import Tables +def _generate_through_passes(sql: str, tables: Tables) -> str: + """Parse, run normalization passes 1 and 2, then generate SQL. + + Coordinate canonicalization for operator operands moved out of the emitter and + into the CanonicalizeCoordinates pass (issue #123). Emitter-level tests that + pin canonicalized output must therefore run both passes before generating, + exactly as :func:`giql.transpile.transpile` does, rather than calling + ``generate`` on a bare parsed AST (which would skip canonicalization). This + helper is used where the full ``transpile`` pipeline would otherwise rewrite + the node away (a column-to-column ``INTERSECTS`` is turned into a binned + equi-join before the predicate emitter runs). + """ + ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables) + ast = canonicalize_coordinates(ast) + return BaseGIQLGenerator(tables=tables).generate(ast) + + @pytest.fixture def tables_info(): """Basic Tables with a single table containing genomic columns.""" @@ -815,11 +836,10 @@ def test_giqldistance_should_not_apply_gap_plus_one_for_closed_intervals( "SELECT DISTANCE(a.interval, b.interval) as dist " "FROM bed_features a CROSS JOIN bed_features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_closed_intervals) - # Act - output = generator.generate(ast) + # Act — canonicalization for DISTANCE operands now lives in the + # CanonicalizeCoordinates pass (#123), so run both passes via the helper. + output = _generate_through_passes(sql, tables_with_closed_intervals) # Assert expected = ( @@ -930,30 +950,29 @@ def test_giqlnearest_should_not_apply_gap_plus_one_for_closed_intervals( Given: A 0-based closed-interval target table and NEAREST. When: - giqlnearest_sql is called. + The query is transpiled. Then: - It should canonicalize the target end as (target."end" + 1) but - omit any "+1" from the gap branches. + It should canonicalize the target end as (end + 1) inside the + wrapper CTE while the distance gap branches read the bare canonical + end with no trailing "+ 1". """ # Arrange - tables = Tables() - tables.register("genes_closed", Table("genes_closed", interval_type="closed")) sql = ( "SELECT * FROM NEAREST(genes_closed, reference := 'chr1:1000-2000', k := 3)" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables) - # Act - output = generator.generate(ast) + # Act — the target's coordinate canonicalization now lives in the + # CanonicalizeCoordinates pass (#123): a non-canonical target is wrapped + # in a __giql_canon_* CTE, so the (end + 1) arithmetic appears there and + # the distance CASE reads the bare canonical columns. + output = transpile(sql, tables=[Table("genes_closed", interval_type="closed")]) - # Assert — canonicalization is applied, but neither gap branch carries - # a trailing "+ 1". - assert '(genes_closed."end" + 1)' in output - assert 'THEN (genes_closed."start" - 2000)' in output - assert 'ELSE (1000 - (genes_closed."end" + 1))' in output - assert '(genes_closed."start" - 2000 + 1)' not in output - assert '(1000 - (genes_closed."end" + 1) + 1)' not in output + # Assert — the wrapper carries (end + 1); the gap branches carry no "+ 1". + assert '("end" + 1) AS "end"' in output + assert 'THEN (__giql_canon_0."start" - 2000)' in output + assert 'ELSE (1000 - __giql_canon_0."end")' in output + assert '__giql_canon_0."start" - 2000 + 1' not in output + assert '1000 - __giql_canon_0."end" + 1' not in output def test_giqldistance_sql_literal_first_arg_error(self, tables_with_two_tables): """ @@ -1272,14 +1291,11 @@ def test_intersects_should_canonicalize_table_columns_for_each_convention( unchanged. """ # Arrange - tables = Tables() - tables.register(table.name, table) sql = "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:100-200'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables) - # Act - output = generator.generate(ast) + # Act — predicate operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. + output = transpile(sql, tables=[table]) # Assert expected = ( @@ -1328,14 +1344,11 @@ def test_contains_should_canonicalize_table_columns_for_each_convention( unchanged. """ # Arrange - tables = Tables() - tables.register(table.name, table) sql = "SELECT * FROM variants WHERE interval CONTAINS 'chr1:1500-2000'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables) - # Act - output = generator.generate(ast) + # Act — predicate operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. + output = transpile(sql, tables=[table]) # Assert expected = ( @@ -1384,14 +1397,11 @@ def test_within_should_canonicalize_table_columns_for_each_convention( unchanged. """ # Arrange - tables = Tables() - tables.register(table.name, table) sql = "SELECT * FROM variants WHERE interval WITHIN 'chr1:1000-5000'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables) - # Act - output = generator.generate(ast) + # Act — predicate operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. + output = transpile(sql, tables=[table]) # Assert expected = ( @@ -1417,11 +1427,13 @@ def test_intersects_should_canonicalize_both_sides_when_conventions_differ( "SELECT * FROM bed_a AS a CROSS JOIN vcf_b AS b " "WHERE a.interval INTERSECTS b.interval" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_mixed_conventions) - # Act - output = generator.generate(ast) + # Act — column-join operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123). The full transpile pipeline rewrites + # a column-to-column INTERSECTS into a binned equi-join before the + # predicate emitter runs, so run passes 1 and 2 directly to exercise the + # predicate emitter's column-join branch on canonicalized metadata. + output = _generate_through_passes(sql, tables_mixed_conventions) # Assert expected = ( @@ -1451,11 +1463,11 @@ def test_contains_should_canonicalize_both_sides_when_conventions_differ( "SELECT * FROM bed_a AS a CROSS JOIN vcf_b AS b " "WHERE a.interval CONTAINS b.interval" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_mixed_conventions) - # Act - output = generator.generate(ast) + # Act — column-join operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. (Column-to-column + # CONTAINS is not rewritten into a binned join, so it reaches the emitter.) + output = transpile(sql, tables=list(tables_mixed_conventions)) # Assert expected = ( @@ -1485,11 +1497,11 @@ def test_within_should_canonicalize_both_sides_when_conventions_differ( "SELECT * FROM bed_a AS a CROSS JOIN vcf_b AS b " "WHERE a.interval WITHIN b.interval" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_mixed_conventions) - # Act - output = generator.generate(ast) + # Act — column-join operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. (Column-to-column + # WITHIN is not rewritten into a binned join, so it reaches the emitter.) + output = transpile(sql, tables=list(tables_mixed_conventions)) # Assert expected = ( @@ -1515,11 +1527,10 @@ def test_contains_point_should_shift_start_when_table_is_one_based_closed( """ # Arrange sql = "SELECT * FROM vcf_variants WHERE interval CONTAINS 'chr1:1500'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_one_based_closed) - # Act - output = generator.generate(ast) + # Act — predicate operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. + output = transpile(sql, tables=list(tables_with_one_based_closed)) # Assert expected = ( @@ -1546,11 +1557,10 @@ def test_intersects_any_should_canonicalize_disjuncts_when_table_is_one_based_cl "SELECT * FROM vcf_variants " "WHERE interval INTERSECTS ANY('chr1:100-200', 'chr1:500-600')" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_one_based_closed) - # Act - output = generator.generate(ast) + # Act — SpatialSetPredicate operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. + output = transpile(sql, tables=list(tables_with_one_based_closed)) # Assert expected = ( @@ -1638,11 +1648,10 @@ def test_giqldistance_should_canonicalize_table_columns_for_each_convention( "SELECT DISTANCE(a.interval, b.interval) as dist " "FROM dist_a a CROSS JOIN dist_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables) - # Act - output = generator.generate(ast) + # Act — DISTANCE operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. + output = transpile(sql, tables=list(tables)) # Assert expected = ( @@ -1673,11 +1682,10 @@ def test_giqldistance_should_canonicalize_each_side_when_conventions_differ( "SELECT DISTANCE(a.interval, b.interval) as dist " "FROM bed_a a CROSS JOIN vcf_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_mixed_conventions) - # Act - output = generator.generate(ast) + # Act — DISTANCE operand canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. + output = transpile(sql, tables=list(tables_mixed_conventions)) # Assert expected = ( @@ -1691,75 +1699,128 @@ def test_giqldistance_should_canonicalize_each_side_when_conventions_differ( assert output == expected @pytest.mark.parametrize( - "coordinate_system, interval_type, target_start, target_end", + "coordinate_system, interval_type, wrap_start, wrap_end", [ - pytest.param( - "0based", - "half_open", - 'genes."start"', - 'genes."end"', - id="0based-half_open", - ), pytest.param( "0based", "closed", - 'genes."start"', - '(genes."end" + 1)', + '"start" AS "start"', + '("end" + 1) AS "end"', id="0based-closed", ), pytest.param( "1based", "half_open", - '(genes."start" - 1)', - '(genes."end" - 1)', + '("start" - 1) AS "start"', + '("end" - 1) AS "end"', id="1based-half_open", ), pytest.param( "1based", "closed", - '(genes."start" - 1)', - 'genes."end"', + '("start" - 1) AS "start"', + '"end" AS "end"', id="1based-closed", ), ], ) def test_giqlnearest_should_canonicalize_target_columns_for_each_convention( - self, coordinate_system, interval_type, target_start, target_end + self, coordinate_system, interval_type, wrap_start, wrap_end ): - """Test NEAREST canonicalizes target endpoints per convention. + """Test NEAREST canonicalizes a non-canonical target via a wrapper CTE. Given: - A target table declared with one of the four (coordinate_system, - interval_type) combinations and a literal reference range. + A target table declared with one of the three non-canonical + (coordinate_system, interval_type) combinations and a literal + reference range. When: - giqlnearest_sql is called. + The query is transpiled. Then: - It should wrap the target-side start/end (or not) per the - canonical 0-based half-open conversion in the distance CASE - expression. + It should wrap the target in a __giql_canon_* CTE carrying the + canonical conversion, and the distance CASE should read the bare + canonical target columns with no in-CASE canonicalization. + """ + # Arrange + sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" + + # Act — the target's coordinate canonicalization now lives in the + # CanonicalizeCoordinates pass (#123): a non-canonical target is wrapped + # in a __giql_canon_* CTE before generation, so the distance CASE reads + # already-canonical columns. + output = transpile( + sql, + tables=[ + Table( + "genes", + coordinate_system=coordinate_system, + interval_type=interval_type, + ) + ], + ) + + # Assert — the wrapper carries the canonical conversion; the distance + # CASE reads the bare canonical columns against the literal [1000, 2000). + assert f"REPLACE ({wrap_start}, {wrap_end}) FROM genes" in output + assert ( + 'WHEN 1000 < __giql_canon_0."end" AND 2000 > __giql_canon_0."start" THEN 0' + ) in output + assert ( + 'WHEN 2000 <= __giql_canon_0."start" THEN (__giql_canon_0."start" - 2000)' + ) in output + assert 'ELSE (1000 - __giql_canon_0."end")' in output + + def test_giqlnearest_should_pass_target_columns_through_when_target_is_canonical( + self, + ): + """Test NEAREST leaves a canonical target unwrapped and byte-identical. + + Given: + A canonical 0-based half-open target table and a literal reference. + When: + The query is transpiled. + Then: + It should synthesize no wrapper CTE; the distance CASE reads the raw + target columns and the row passes through as a plain ``genes.*``. """ # Arrange - tables = Tables() - tables.register( - "genes", - Table( - "genes", - coordinate_system=coordinate_system, - interval_type=interval_type, - ), - ) sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables) # Act - output = generator.generate(ast) + output = transpile(sql, tables=[Table("genes")]) + + # Assert + assert "__giql_canon_" not in output + assert "genes.*," in output + assert 'WHEN 1000 < genes."end" AND 2000 > genes."start" THEN 0' in output + + def test_giqlnearest_should_round_trip_passthrough_row_to_target_encoding(self): + """Test NEAREST de-canonicalizes the passed-through row to the target encoding. - # Assert — distance CASE expression uses canonicalized target endpoints - # against the (already-canonical) literal reference [1000, 2000). - assert f"WHEN 1000 < {target_end} AND 2000 > {target_start} THEN 0" in output - assert f"WHEN 2000 <= {target_start} THEN ({target_start} - 2000)" in output - assert f"ELSE (1000 - {target_end})" in output + Given: + A 1-based closed target table wrapped by the canonicalization pass. + When: + The query is transpiled. + Then: + The ``*`` passthrough should de-canonicalize the interval columns + back to the target's declared encoding via a star REPLACE, so the + returned row carries the table's own convention; the synthesized + ``distance`` column is encoding-invariant and stays unwrapped. + """ + # Arrange + sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" + + # Act + output = transpile( + sql, + tables=[Table("genes", coordinate_system="1based", interval_type="closed")], + ) + + # Assert — passthrough de-canonicalizes 1-based-closed start as (start + 1) + # and leaves end identity; the distance column carries no round-trip. + assert ( + '__giql_canon_0.* REPLACE ((__giql_canon_0."start" + 1) AS "start", ' + '__giql_canon_0."end" AS "end")' + ) in output def test_giqlnearest_should_canonicalize_reference_column_when_reference_is_one_based_closed( self, tables_mixed_conventions @@ -1780,13 +1841,14 @@ def test_giqlnearest_should_canonicalize_reference_column_when_reference_is_one_ "SELECT * FROM vcf_b b CROSS JOIN LATERAL " "NEAREST(bed_a, reference := b.interval, k := 1)" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_mixed_conventions) - # Act - output = generator.generate(ast) + # Act — the reference operand's canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. The reference is + # a column operand (not a wrappable relation), so the arithmetic stays + # inline; the canonical target bed_a is left unwrapped. + output = transpile(sql, tables=list(tables_mixed_conventions)) - # Assert — reference's start canonicalized via _canonical_start + # Assert — reference's start canonicalized as (start - 1) assert '(b."start" - 1) < bed_a."end"' in output assert 'bed_a."end" <= (b."start" - 1)' not in output # reference is left side # Reference's end stays raw (1-based closed → identity for end) @@ -1810,11 +1872,12 @@ def test_giqlnearest_should_canonicalize_outer_table_columns_when_reference_is_i """ # Arrange sql = "SELECT * FROM vcf_b CROSS JOIN LATERAL NEAREST(bed_a, k := 1)" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_mixed_conventions) - # Act - output = generator.generate(ast) + # Act — the implicit-outer reference's canonicalization now lives in the + # CanonicalizeCoordinates pass (#123); transpile runs it. The outer + # reference is a column operand (not a wrappable relation), so the + # arithmetic stays inline; the canonical target bed_a is left unwrapped. + output = transpile(sql, tables=list(tables_mixed_conventions)) # Assert — distance CASE uses canonicalized outer-table columns and # raw target-table columns. diff --git a/tests/test_canonicalizer.py b/tests/test_canonicalizer.py index 6899e9a..afea3e5 100644 --- a/tests/test_canonicalizer.py +++ b/tests/test_canonicalizer.py @@ -22,6 +22,9 @@ from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expressions import GIQLDisjoin +from giql.expressions import GIQLDistance +from giql.expressions import GIQLNearest +from giql.expressions import Intersects from giql.generators import BaseGIQLGenerator from giql.resolver import META_KEY from giql.resolver import resolve_operator_refs @@ -631,3 +634,193 @@ def test_all_encoding_pairs_covered(): # Assert assert len(pairs) == 4 assert len(canonical) == 1 + + +def _two_tables(encoding) -> list[Table]: + """Build two registered tables under the same (non-default) encoding.""" + coordinate_system, interval_type = encoding + return [ + Table(name, coordinate_system=coordinate_system, interval_type=interval_type) + for name in ("intervals_a", "intervals_b") + ] + + +class TestColumnOperandCanonicalization: + """Pass 2 canonicalizes column / interval operands in place (issue #123). + + A column operand references an alias bound in the enclosing query's FROM, + shared with the user's own projection, so it cannot be wrapped in a canonical + CTE without changing unrelated columns. Pass 2 therefore rewrites the operand's + resolution metadata to carry the canonical arithmetic inline and the emitter + consumes it verbatim — no in-emitter canonicalization, no wrapper CTE. + """ + + def test_distance_operand_canonicalized_inline_without_wrapper_cte(self): + """Test a non-canonical DISTANCE operand canonicalizes inline, no wrapper. + + Given: + Two 1-based closed tables and a DISTANCE between their columns. + When: + The query is transpiled. + Then: + The CASE arithmetic should wrap each side's start as (start - 1) + inline and synthesize no __giql_canon_* wrapper CTE. + """ + # Arrange + sql = ( + "SELECT DISTANCE(a.interval, b.interval) AS dist " + "FROM intervals_a a, intervals_b b" + ) + + # Act + output = transpile(sql, tables=_two_tables(("1based", "closed"))) + + # Assert + assert CANON_PREFIX not in output + assert '(a."start" - 1)' in output + assert '(b."start" - 1)' in output + + def test_canonical_distance_operand_is_byte_identical(self): + """Test a canonical DISTANCE operand emits byte-identical SQL. + + Given: + Two default (0-based half-open) tables and a DISTANCE between their + columns, transpiled with the operator opted in and with its + GIQL_CANONICALIZE flag toggled off. + When: + The two transpilations are compared. + Then: + They should be byte-identical and carry no wrapper CTE — the pass is + inert for an already-canonical operand. + """ + # Arrange + sql = ( + "SELECT DISTANCE(a.interval, b.interval) AS dist " + "FROM intervals_a a, intervals_b b" + ) + tables = [Table("intervals_a"), Table("intervals_b")] + + # Act + opted_in = transpile(sql, tables=tables) + previous = GIQLDistance.GIQL_CANONICALIZE + GIQLDistance.GIQL_CANONICALIZE = False + try: + flag_off = transpile(sql, tables=tables) + finally: + GIQLDistance.GIQL_CANONICALIZE = previous + + # Assert + assert opted_in == flag_off + assert CANON_PREFIX not in opted_in + + def test_predicate_operand_canonicalized_inline_without_wrapper_cte(self): + """Test a non-canonical INTERSECTS operand canonicalizes inline, no wrapper. + + Given: + A 1-based closed table and a literal-range INTERSECTS predicate. + When: + The query is transpiled. + Then: + The predicate should wrap the table-side start as (start - 1) inline + and synthesize no __giql_canon_* wrapper CTE. + """ + # Arrange + sql = "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:100-200'" + + # Act + output = transpile( + sql, + tables=[ + Table("variants", coordinate_system="1based", interval_type="closed") + ], + ) + + # Assert + assert CANON_PREFIX not in output + assert '("start" - 1) < 200' in output + + def test_metadata_blanked_after_in_place_canonicalization(self): + """Test a canonicalized column operand carries a blanked Table. + + Given: + A 1-based closed INTERSECTS predicate annotated by pass 1. + When: + Pass 2 runs. + Then: + The operand's ResolvedColumn should carry the canonical arithmetic and + its Table should be blanked so the emitter applies no further wrapping. + """ + # Arrange + tables = _tables(("1based", "closed"), names=("variants",)) + query = "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:100-200'" + ast = resolve_operator_refs(parse_one(query, dialect=GIQLDialect), tables) + + # Act + ast = canonicalize_coordinates(ast) + + # Assert + node = next(n for n in ast.walk() if isinstance(n, Intersects)) + column = node.meta[META_KEY].column("this") + assert column.table is None + assert column.start == '("start" - 1)' + + +class TestNearestTargetCanonicalization: + """NEAREST's registered-table target is wrapped and its row round-trips.""" + + def test_non_canonical_target_wrapped_and_row_round_tripped(self): + """Test a non-canonical NEAREST target is wrapped and its row de-canonicalized. + + Given: + A 1-based closed NEAREST target table and a literal reference. + When: + The query is transpiled. + Then: + The target should be wrapped in a __giql_canon_* CTE, the distance + CASE should read the bare canonical columns, and the passed-through + row should de-canonicalize the interval back to the declared encoding. + """ + # Arrange + sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" + + # Act + output = transpile( + sql, + tables=[Table("genes", coordinate_system="1based", interval_type="closed")], + ) + + # Assert + assert f"{CANON_PREFIX}0 AS (SELECT * REPLACE" in output + assert 'WHEN 1000 < __giql_canon_0."end"' in output + assert ( + '__giql_canon_0.* REPLACE ((__giql_canon_0."start" + 1) AS "start"' + ) in output + + def test_canonical_target_not_wrapped(self): + """Test a canonical NEAREST target is left unwrapped. + + Given: + A canonical 0-based half-open NEAREST target and a literal reference, + transpiled with NEAREST opted in and with its flag toggled off. + When: + The two transpilations are compared. + Then: + They should be byte-identical with no wrapper CTE — the identity fast + path. + """ + # Arrange + sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" + tables = [Table("genes")] + + # Act + opted_in = transpile(sql, tables=tables) + previous = GIQLNearest.GIQL_CANONICALIZE + GIQLNearest.GIQL_CANONICALIZE = False + try: + flag_off = transpile(sql, tables=tables) + finally: + GIQLNearest.GIQL_CANONICALIZE = previous + + # Assert + assert opted_in == flag_off + assert CANON_PREFIX not in opted_in From 0224849187e75e97c8942bd13e47c69278f5f2f0 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 18:43:51 -0400 Subject: [PATCH 050/142] feat: Validate CTE and subquery projection contracts in pass 1 Add a user-facing validation boundary to the resolver pass that checks every CTE or subquery operand actually projects the canonical chrom, start, and end columns the operator will reference. A malformed operand now raises a clear GIQL ValueError at transpile time -- naming the operator, the relation, the missing columns, and a source position -- instead of surfacing as an opaque engine column-not-found error at execution time. The check only fires when the operand's output schema is statically known: a projection of intentional named columns. Star projections and unnamed or computed projections (SELECT *, SELECT 1, SELECT chrom + 1) are treated as unresolvable and pass through, so the contract is never falsely reported when GIQL cannot see the columns. Set operations defer to their left branch, matching SQL column-name semantics. The boundary is additive and side-effect-free, raising through the existing resolution error boundary in transpile. --- src/giql/resolver.py | 194 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) diff --git a/src/giql/resolver.py b/src/giql/resolver.py index 9fa3104..80d0969 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -97,6 +97,7 @@ "ResolutionError", "resolve_operator_refs", "validate_operator_refs", + "validate_projection_contracts", ] #: The single namespaced key under which resolution metadata is stored on a @@ -419,6 +420,7 @@ def resolve_operator_refs(expression: exp.Expression, tables: Tables) -> exp.Exp _resolve_operator(node, tables, frozenset()) validate_operator_refs(expression) + validate_projection_contracts(expression) return expression @@ -1157,3 +1159,195 @@ def _validate_ref(ref: object, spec: SlotSpec, operator: str) -> None: f"{operator} slot {spec.arg!r} resolved to a {ref.kind} but carries " "a Table config; CTE and subquery references are assumed canonical." ) + + +def validate_projection_contracts(expression: exp.Expression) -> None: + """Assert every CTE / subquery operator operand projects canonical columns. + + A user-facing GIQL diagnostic boundary (epic #114, step 9). When an operator + reference slot resolves to an in-query CTE or a ``(SELECT ...)`` subquery + (:class:`ResolvedRef` kind ``"cte"`` / ``"subquery"``), the resolver and the + canonicalizer both *assume* it exposes the canonical ``chrom`` / ``start`` / + ``end`` columns the operator will reference — an assumption that, when wrong, + historically surfaced only as an opaque engine ``column not found`` error at + execution time. This pass inspects the operand's projection at transpile time + and raises a clear :class:`ValueError` naming the operator, the offending + relation, and the missing column(s) instead. + + Unlike :func:`validate_operator_refs` (an internal invariant check that raises + :class:`ResolutionError`), this is a *user-facing* check that raises + :class:`ValueError` so it surfaces verbatim through ``transpile()``'s + "Resolution error" boundary. + + False-positive guard + -------------------- + A projection's output column names are only statically known when **every** + projection expression is an intentional named column — a bare ``exp.Column`` + (whose output name is the column) or an aliased expression ``exp.Alias`` + (whose output name is the explicit alias). If any projection is a ``*`` star + (whose columns expand from an unknown source schema) or an unnamed expression + (a bare literal like ``SELECT 1`` or a computed ``chrom + 1`` with no alias), + the relation's schema is not statically determinable and the operand is + **passed** without a diagnostic — a documented limitation that avoids + false-positives on star and placeholder projections. The check fires only + when the schema is fully known *and* a canonical column is provably absent. + + Parameters + ---------- + expression : exp.Expression + The pass-1-annotated AST to validate. + + Raises + ------ + ValueError + If a CTE or subquery operand's statically-known projection omits one or + more of the canonical ``chrom`` / ``start`` / ``end`` columns the + operator references. + """ + cte_bodies = _cte_body_index(expression) + for node in expression.walk(): + if not isinstance(node, _OPERATORS): + continue + resolution = node.meta.get(META_KEY) + if not isinstance(resolution, OperatorResolution): + continue + operator = resolution.operator + for arg, ref in resolution.slots.items(): + if not isinstance(ref, ResolvedRef): + continue + if ref.kind not in ("cte", "subquery"): + continue + select = _projection_select(node, arg, ref, cte_bodies) + if select is None: + continue + _check_projection(node, arg, ref, select, operator) + + +def _cte_body_index(expression: exp.Expression) -> dict[str, exp.Expression]: + """Map each in-query CTE alias to the query body it defines. + + Built once per validation so a CTE-kind reference can locate its ``WITH`` + definition and read its output projection. The body is the CTE's ``this`` — + an ``exp.Select`` or set operation (``exp.Union`` / ``exp.Intersect`` / + ``exp.Except``). When two CTEs share an alias (inner-WITH redeclaration), the + last one wins; the resolver's scope-based reference resolution already + selected which one the operator sees, and the projection contract is + identical across redeclarations for the columns we check. + """ + index: dict[str, exp.Expression] = {} + for cte in expression.find_all(exp.CTE): + body = cte.this + if body is not None: + index[cte.alias] = body + return index + + +def _projection_select( + node: exp.Expression, + arg: str, + ref: ResolvedRef, + cte_bodies: dict[str, exp.Expression], +) -> exp.Expression | None: + """Return the query body whose projection backs reference slot *arg*. + + For a CTE reference the body is looked up by name in *cte_bodies*; for a + subquery reference it is the operand AST node itself (an ``exp.Subquery`` + unwrapped to its inner query, or a bare ``exp.Select`` / set operation). + Returns ``None`` when no projection-bearing body can be located — e.g. a CTE + whose definition is not in the index (out of scope) — so the caller skips it. + """ + if ref.kind == "cte": + return cte_bodies.get(ref.name) if ref.name else None + operand = node.args.get(arg) + if isinstance(operand, exp.Subquery): + operand = operand.this + if isinstance(operand, (exp.Select, exp.SetOperation)): + return operand + return None + + +def _static_output_columns(select: exp.Expression) -> frozenset[str] | None: + """Return a body's statically-known output column names, or ``None``. + + ``None`` signals an *unresolvable* projection — one whose output schema + cannot be determined statically (a ``*`` star or any unnamed expression) — + which the contract check treats as a pass (the false-positive guard). A + non-``None`` result is the complete set of intentional output column names, + safe to test for the canonical columns. + + A set operation (``UNION`` / ``INTERSECT`` / ``EXCEPT``) defers to its left + branch, whose projection determines the result's column names in SQL. + """ + if isinstance(select, exp.SetOperation): + return _static_output_columns(select.this) + if not isinstance(select, exp.Select): + return None + + names: set[str] = set() + for projection in select.selects: + if not isinstance(projection, (exp.Column, exp.Alias)): + # A star or an unnamed expression (bare literal, computed column): + # the output schema is not statically determinable, so the whole + # projection is unresolvable and the operand passes. + return None + if isinstance(projection, exp.Column) and isinstance(projection.this, exp.Star): + # A qualified star (``a.*``) also expands from an unknown schema. + return None + names.add(projection.output_name) + return frozenset(names) + + +def _check_projection( + node: exp.Expression, + arg: str, + ref: ResolvedRef, + select: exp.Expression, + operator: str, +) -> None: + """Raise a GIQL diagnostic if *select* omits a canonical column. + + Compares the operand's statically-known output columns against the canonical + column names the operator references (``ref.cols``). A star or otherwise + unresolvable projection (``_static_output_columns`` returns ``None``) is a + pass. When the schema is known and any required column is absent, raises a + :class:`ValueError` naming the operator, the offending relation, the missing + column(s), and a source position when one is available. + """ + columns = _static_output_columns(select) + if columns is None: + return + missing = [col for col in ref.cols if col not in columns] + if not missing: + return + + relation = f"CTE {ref.name!r}" if ref.kind == "cte" else "subquery" + missing_str = ", ".join(repr(col) for col in missing) + have_str = ", ".join(repr(col) for col in sorted(columns)) or "no columns" + position = _source_position(node.args.get(arg)) + raise ValueError( + f"{operator} {arg} {relation} does not project the canonical " + f"genomic column(s) {missing_str} that the operator references; " + f"it projects {have_str}. A CTE or subquery operand must project " + f"canonical 0-based half-open 'chrom'/'start'/'end' columns." + f"{position}" + ) + + +def _source_position(operand: exp.Expression | None) -> str: + """Return a `` (line L, col C)`` suffix for *operand*, or ``""``. + + Mirrors ``sqlglot``'s ``validate_qualify_columns`` precedent of attaching the + parser-populated source position to a validation diagnostic. The position is + best-effort: when the parser did not record one (``meta`` carries no + ``line``), an empty string is returned and the textual diagnostic stands on + its own. + """ + if operand is None: + return "" + line = operand.meta.get("line") + col = operand.meta.get("col") + if line is None: + return "" + if col is None: + return f" (line {line})" + return f" (line {line}, col {col})" From 39bb617fb1427d0bee78255939888ef8efc6f764 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 18:43:51 -0400 Subject: [PATCH 051/142] test: Cover CTE and subquery projection-contract validation Pin that a CTE or subquery operand missing a canonical column raises the GIQL diagnostic, that a well-formed operand resolves cleanly, that star and placeholder projections pass without a false positive, and that the diagnostic surfaces through transpile. --- tests/test_resolver.py | 243 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 243 insertions(+) diff --git a/tests/test_resolver.py b/tests/test_resolver.py index c92afae..4035d98 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -26,8 +26,10 @@ from giql.resolver import ResolvedRef from giql.resolver import resolve_operator_refs from giql.resolver import validate_operator_refs +from giql.resolver import validate_projection_contracts from giql.table import Table from giql.table import Tables +from giql.transpile import transpile hypothesis = pytest.importorskip("hypothesis") from hypothesis import given # noqa: E402 @@ -1396,3 +1398,244 @@ def test_validate_operator_refs_rejects_non_resolvedinterval_slot_value(self): # Act & assert with pytest.raises(ResolutionError, match="expected ResolvedInterval"): validate_operator_refs(ast) + + +class TestValidateProjectionContracts: + """Tests for the validate_projection_contracts pass.""" + + def test_validate_projection_contracts_cte_missing_column_raises(self): + """Test that a CTE operand missing a canonical column is rejected. + + Given: + A DISJOIN reference naming a CTE whose projection lists only + ``chrom`` and ``start``, omitting the canonical ``end`` column. + When: + Running the resolve pass over the annotated AST. + Then: + It should raise a ValueError naming the operator, the CTE, and the + missing ``end`` column. + """ + # Arrange + ast = parse_one( + "WITH refs AS (SELECT chrom, start FROM other) " + "SELECT * FROM DISJOIN(features, reference := refs)", + dialect=GIQLDialect, + ) + + # Act & assert + with pytest.raises(ValueError, match=r"GIQLDisjoin reference CTE 'refs'.*'end'"): + resolve_operator_refs(ast, _tables("features", "other")) + + def test_validate_projection_contracts_subquery_missing_column_raises(self): + """Test that a subquery operand missing a canonical column is rejected. + + Given: + A DISJOIN reference subquery whose projection lists only ``start`` + and ``end``, omitting the canonical ``chrom`` column. + When: + Running the resolve pass over the annotated AST. + Then: + It should raise a ValueError naming the operator, the subquery, and + the missing ``chrom`` column. + """ + # Arrange + ast = parse_one( + "SELECT * FROM DISJOIN(features, " + 'reference := (SELECT start, "end" FROM other))', + dialect=GIQLDialect, + ) + + # Act & assert + with pytest.raises(ValueError, match=r"GIQLDisjoin reference subquery.*'chrom'"): + resolve_operator_refs(ast, _tables("features", "other")) + + def test_validate_projection_contracts_well_formed_cte_passes(self): + """Test that a CTE projecting all canonical columns is accepted. + + Given: + A DISJOIN reference naming a CTE whose projection lists all three + canonical ``chrom`` / ``start`` / ``end`` columns. + When: + Running the resolve pass over the annotated AST. + Then: + It should resolve the reference to a CTE without raising. + """ + # Arrange + ast = parse_one( + 'WITH refs AS (SELECT chrom, start, "end" FROM other) ' + "SELECT * FROM DISJOIN(features, reference := refs)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features", "other")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "cte" + + def test_validate_projection_contracts_well_formed_subquery_passes(self): + """Test that a subquery projecting all canonical columns is accepted. + + Given: + A DISJOIN reference subquery whose projection lists all three + canonical ``chrom`` / ``start`` / ``end`` columns. + When: + Running the resolve pass over the annotated AST. + Then: + It should resolve the reference to a subquery without raising. + """ + # Arrange + ast = parse_one( + "SELECT * FROM DISJOIN(features, " + 'reference := (SELECT chrom, start, "end" FROM other))', + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features", "other")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "subquery" + + def test_validate_projection_contracts_star_cte_passes(self): + """Test that a ``SELECT *`` CTE is passed without a false positive. + + Given: + A DISJOIN reference naming a CTE whose projection is a ``*`` star, + so its output columns are not statically known. + When: + Running the resolve pass over the annotated AST. + Then: + It should resolve the reference without raising (documented star + limitation). + """ + # Arrange + ast = parse_one( + "WITH refs AS (SELECT * FROM other) " + "SELECT * FROM DISJOIN(features, reference := refs)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features", "other")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "cte" + + def test_validate_projection_contracts_star_subquery_passes(self): + """Test that a ``SELECT *`` subquery is passed without a false positive. + + Given: + A DISJOIN reference subquery whose projection is a ``*`` star, so its + output columns are not statically known. + When: + Running the resolve pass over the annotated AST. + Then: + It should resolve the reference without raising (documented star + limitation). + """ + # Arrange + ast = parse_one( + "SELECT * FROM DISJOIN(features, reference := (SELECT * FROM other))", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features", "other")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "subquery" + + def test_validate_projection_contracts_placeholder_cte_passes(self): + """Test that a bare-literal placeholder CTE is passed. + + Given: + A DISJOIN reference naming a CTE whose projection is the placeholder + ``SELECT 1`` -- an unnamed literal with no statically intentional + column schema. + When: + Running the resolve pass over the annotated AST. + Then: + It should resolve the reference without raising (unresolvable + projection limitation). + """ + # Arrange + ast = parse_one( + "WITH refs AS (SELECT 1) SELECT * FROM DISJOIN(features, reference := refs)", + dialect=GIQLDialect, + ) + + # Act + resolve_operator_refs(ast, _tables("features", "other")) + + # Assert + ref = _disjoin_node(ast).meta[META_KEY].slot("reference") + assert ref.kind == "cte" + + def test_validate_projection_contracts_can_run_standalone(self): + """Test that the contract check runs as a standalone boundary. + + Given: + An AST already annotated by the resolve pass whose DISJOIN reference + CTE omits the canonical ``end`` column. + When: + Calling ``validate_projection_contracts`` directly on the annotated + AST. + Then: + It should raise the GIQL diagnostic independently of the surrounding + resolve pass. + """ + # Arrange + ast = parse_one( + "WITH refs AS (SELECT chrom, start FROM other) " + "SELECT * FROM DISJOIN(features, reference := refs)", + dialect=GIQLDialect, + ) + node = _disjoin_node(ast) + node.meta[META_KEY] = OperatorResolution( + "GIQLDisjoin", + { + "this": ResolvedRef( + kind="registered_table", + name="features", + cols=("chrom", "start", "end"), + table=Table("features"), + coverage_skippable=False, + ), + "reference": ResolvedRef( + kind="cte", + name="refs", + cols=("chrom", "start", "end"), + table=None, + coverage_skippable=False, + ), + }, + ) + + # Act & assert + with pytest.raises(ValueError, match="does not project the canonical"): + validate_projection_contracts(ast) + + def test_validate_projection_contracts_fires_through_transpile(self): + """Test that the diagnostic surfaces through ``transpile``. + + Given: + A GIQL query whose DISJOIN reference CTE omits the canonical ``end`` + column. + When: + Transpiling the query. + Then: + It should raise a ValueError carrying the GIQL-level diagnostic + rather than emitting SQL that fails at execution time. + """ + # Arrange, act, & assert + with pytest.raises(ValueError, match=r"GIQLDisjoin reference CTE 'refs'.*'end'"): + transpile( + "WITH refs AS (SELECT chrom, start FROM other) " + "SELECT * FROM DISJOIN(features, reference := refs)", + tables=["features", "other"], + ) From c505dc9320278115f4ba8b0947aec0ff0db16cae Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 10 Jun 2026 18:43:51 -0400 Subject: [PATCH 052/142] docs: Document the CTE and subquery reference contract Describe the requirement that a CTE or subquery operand project canonical 0-based half-open chrom, start, and end columns, and note that GIQL now validates this at transpile time rather than deferring to an opaque engine error. Update the schema-mapping note that previously described the CTE contract as assumed-canonical and unvalidated. --- docs/recipes/disjoin.rst | 30 +++++++++++++++++++++++++++ docs/transpilation/schema-mapping.rst | 10 ++++++--- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/docs/recipes/disjoin.rst b/docs/recipes/disjoin.rst index f6a4d7b..810416b 100644 --- a/docs/recipes/disjoin.rst +++ b/docs/recipes/disjoin.rst @@ -80,6 +80,36 @@ single bin. engines need their own generator. The grid must also span every chromosome present in ``features``, or features on an uncovered chromosome are dropped. +The CTE / Subquery Reference Contract +------------------------------------- + +When the ``reference`` operand is an in-query CTE or a ``(SELECT ...)`` +subquery (rather than a registered table), it **must** project the canonical +``chrom`` / ``start`` / ``end`` columns -- the 0-based, half-open genomic +coordinates the operator references. A registered table may declare a custom +column mapping or coordinate system, but a CTE or subquery carries no such +declaration, so GIQL assumes it is already canonical. + +GIQL validates this contract at transpile time. If a CTE or subquery operand +omits one of the canonical columns, ``transpile()`` raises a ``ValueError`` +naming the operator, the offending relation, and the missing column(s) -- a +clear GIQL-level diagnostic rather than an opaque engine ``column not found`` +error at execution time. For example, ``reference := (SELECT chrom, start FROM +mask)`` is rejected because it omits ``end``. + +.. code-block:: sql + + -- Valid: the subquery projects all three canonical columns + SELECT * FROM DISJOIN(features, reference := ( + SELECT chrom, start, "end" FROM mask WHERE score > 10 + )) + +This check is best-effort. When the operand's output columns cannot be +determined statically -- a ``SELECT *`` star projection, or a placeholder such +as ``SELECT 1`` -- the operand is passed through without a diagnostic (the +relation's schema is unknown at transpile time), and a genuinely missing column +would still surface as an engine error at execution time. + Coming from Bedtools? --------------------- diff --git a/docs/transpilation/schema-mapping.rst b/docs/transpilation/schema-mapping.rst index 2cba2ea..3dbcdef 100644 --- a/docs/transpilation/schema-mapping.rst +++ b/docs/transpilation/schema-mapping.rst @@ -186,9 +186,13 @@ If your data uses 1-based coordinates (like VCF or GFF), configure the To target a non-``REPLACE`` engine today, store your data in 0-based half-open form, or convert it explicitly in a CTE and reference that CTE - (which GIQL treats as already canonical). Making canonicalization emit - portable SQL on every engine is tracked in - `#132 `_. + (which GIQL treats as already canonical). Such a CTE -- and any CTE or + subquery passed as an operator reference -- must project the canonical + ``chrom`` / ``start`` / ``end`` columns; GIQL validates this contract at + transpile time and raises a ``ValueError`` naming the missing column(s) + rather than emitting SQL that fails with an engine ``column not found`` + error. Making canonicalization emit portable SQL on every engine is tracked + in `#132 `_. Working with Point Features ~~~~~~~~~~~~~~~~~~~~~~~~~~~ From c4ce83d8b3a79a72b7f15cf7bef9e787868572cd Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 10:10:07 -0400 Subject: [PATCH 053/142] refactor: Remove the generator's legacy resolution fallback path Delete the resolution code in BaseGIQLGenerator that the operator ports superseded: _get_column_refs, _resolve_table, _resolve_table_name, the select_sql override, and its _alias_to_table / _current_table tracking, along with the now-unused imports. These were reachable only through the direct-generate() fallback the operator emitters carried during the migration; the emitters now read exclusively the metadata attached by the resolver pass, raising the historical diagnostics for genuinely unresolvable operands. The generator no longer performs name resolution, scope walks, or input coordinate canonicalization. Output de-canonicalization (decanonical_* on the synthesized disjoin and NEAREST passthrough columns) and the per-node string-expansion methods stay, deferred to the operator- expansion-to-AST work. --- src/giql/generators/base.py | 220 ++++-------------------------------- 1 file changed, 23 insertions(+), 197 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 1874094..84dfd34 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -3,10 +3,6 @@ from giql.canonical import decanonical_end from giql.canonical import decanonical_start -from giql.constants import DEFAULT_CHROM_COL -from giql.constants import DEFAULT_END_COL -from giql.constants import DEFAULT_START_COL -from giql.constants import DEFAULT_STRAND_COL from giql.expressions import Contains from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance @@ -21,7 +17,6 @@ from giql.resolver import ResolvedColumn from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef -from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables @@ -46,40 +41,6 @@ class BaseGIQLGenerator(Generator): def __init__(self, tables: Tables | None = None, **kwargs): super().__init__(**kwargs) self.tables = tables or Tables() - self._current_table = None # Track current table for column resolution - self._alias_to_table = {} # Map aliases to table names - - def select_sql(self, expression: exp.Select) -> str: - """Override SELECT generation to track table context and aliases.""" - # Build alias-to-table mapping - self._alias_to_table = {} - - # Extract from FROM clause - if expression.args.get("from_"): - from_clause = expression.args["from_"] - if isinstance(from_clause.this, exp.Table): - table_name = from_clause.this.name - self._current_table = table_name - # Check if table has an alias - if from_clause.this.alias: - self._alias_to_table[from_clause.this.alias] = table_name - else: - # No alias, table referenced by name - self._alias_to_table[table_name] = table_name - - # Extract from JOINs - if expression.args.get("joins"): - for join in expression.args["joins"]: - if isinstance(join.this, exp.Table): - table_name = join.this.name - # Check if table has an alias - if join.this.alias: - self._alias_to_table[join.this.alias] = table_name - else: - self._alias_to_table[table_name] = table_name - - # Call parent implementation - return super().select_sql(expression) def intersects_sql(self, expression: Intersects) -> str: """Generate standard SQL for INTERSECTS. @@ -537,13 +498,12 @@ def giqldistance_sql(self, expression: GIQLDistance) -> str: Reads the :class:`~giql.resolver.ResolvedColumn` metadata that ``ResolveOperatorRefs`` (pass 1) attaches to each interval operand. When - the pass deferred an operand (a literal range, an unqualified column, or - a tree the pass never reached) the emitter falls back to its historical - string-level resolution and raises the same diagnostics as before. + the pass deferred an operand (a literal range, or an unqualified column) + the emitter raises the historical literal-range diagnostic. - Coordinate canonicalization stays here (epic #114 step 8 / issue #123): - the resolved metadata carries each operand's :class:`~giql.table.Table` - so the endpoints are wrapped identically. + Coordinate canonicalization is owned by ``CanonicalizeCoordinates`` + (pass 2, issue #123): the resolved metadata's endpoints are already + canonicalized in place, so the emitter consumes them verbatim. :param expression: GIQLDistance expression node @@ -585,14 +545,12 @@ def giqldistance_sql(self, expression: GIQLDistance) -> str: def _distance_operand( self, expression: GIQLDistance, arg: str, position: str ) -> ResolvedColumn: - """Resolve one DISTANCE interval operand to a :class:`ResolvedColumn`. + """Return the :class:`ResolvedColumn` for one DISTANCE interval operand. - Prefers the metadata attached by ``ResolveOperatorRefs`` (pass 1). When - the pass deferred the operand — it could not resolve a literal range or - an unqualified column, or never ran (the generator was invoked directly - without the pass) — this falls back to the legacy ``_get_column_refs`` / - ``_resolve_table`` path, raising the historical literal-range error for - a non-column operand. + Reads the metadata attached by ``ResolveOperatorRefs`` (pass 1). When the + pass deferred the operand — a literal range or an unqualified column it + could not resolve — no column is attached and this raises the historical + literal-range diagnostic. :param expression: GIQLDistance expression node @@ -612,19 +570,6 @@ def _distance_operand( if resolved is not None: return resolved - # Deferred: fall back to string-level resolution. - operand_sql = self.sql(expression.args.get(arg)) - if "." in operand_sql and not operand_sql.startswith("'"): - chrom, start, end, strand = self._get_column_refs( - operand_sql, None, include_strand=True - ) - return ResolvedColumn( - chrom=chrom, - start=start, - end=end, - strand=strand, - table=self._resolve_table(operand_sql), - ) raise ValueError(f"Literal range as {position} argument not yet supported") def _generate_distance_case( @@ -718,28 +663,17 @@ def _generate_distance_case( f"ELSE ({start_a} - {end_b}) END END" ) - def _predicate_operand( - self, expression: exp.Expression, arg: str, ctx_table: str | None - ) -> ResolvedColumn: + def _predicate_operand(self, expression: exp.Expression, arg: str) -> ResolvedColumn: """Return the :class:`ResolvedColumn` for a spatial predicate operand. Reads the column resolution attached to *expression* by the - ``ResolveOperatorRefs`` pass (the metadata-driven path used by the full - transpile pipeline). When the pass did not annotate the node — e.g. a - generator invoked on a bare AST without running pass 1 — it falls back to - the generator's historical ``_current_table`` / alias-map resolution so - direct ``generate()`` callers keep their existing behavior. Both paths - format physical column references identically, so the emitted SQL is the - same regardless of which produced the :class:`ResolvedColumn`. + ``ResolveOperatorRefs`` pass (pass 1). The emitter consumes only the + resolved metadata; all name/column resolution lives in the pass. :param expression: The spatial predicate node carrying the resolution metadata. :param arg: The operand slot key (``"this"`` or ``"expression"``). - :param ctx_table: - The current-table resolution context for the fallback path — - ``self._current_table`` for a literal-range operand, ``None`` for a - column-to-column operand. :return: The resolved column metadata. """ @@ -749,10 +683,10 @@ def _predicate_operand( if resolved is not None: return resolved - column_ref = self.sql(expression, arg) - chrom, start, end = self._get_column_refs(column_ref, ctx_table) - table = self._resolve_table(column_ref, ctx_table) - return ResolvedColumn(chrom=chrom, start=start, end=end, strand="", table=table) + raise ValueError( + f"Spatial predicate operand {arg!r} was not resolved; run the " + "ResolveOperatorRefs pass (transpile pipeline) before generation." + ) def _generate_spatial_op(self, expression: exp.Binary, op_type: str) -> str: """Generate SQL for a spatial operation. @@ -769,15 +703,15 @@ def _generate_spatial_op(self, expression: exp.Binary, op_type: str) -> str: # Check if right side is a column reference or a literal range string if "." in right_raw and not right_raw.startswith("'"): # Column-to-column join (e.g., a.interval INTERSECTS b.interval) - left = self._predicate_operand(expression, "this", None) - right = self._predicate_operand(expression, "expression", None) + left = self._predicate_operand(expression, "this") + right = self._predicate_operand(expression, "expression") return self._generate_column_join(left, right, op_type) else: # Literal range string (e.g., interval INTERSECTS 'chr1:1000-2000') try: range_str = right_raw.strip("'\"") parsed_range = RangeParser.parse(range_str).to_zero_based_half_open() - left = self._predicate_operand(expression, "this", self._current_table) + left = self._predicate_operand(expression, "this") return self._generate_range_predicate(left, parsed_range, op_type) except Exception as e: raise ValueError( @@ -919,7 +853,7 @@ def _generate_spatial_set(self, expression: SpatialSetPredicate) -> str: # Resolve the (single) left column operand once; every range condition # compares against the same column. The set predicate's ranges are # always literals, so only this operand needs resolution. - column = self._predicate_operand(expression, "this", self._current_table) + column = self._predicate_operand(expression, "this") # Parse all ranges parsed_ranges = [] @@ -970,10 +904,8 @@ def _nearest_resolution(self, expression: GIQLNearest) -> OperatorResolution | N The transpile pipeline attaches an :class:`~giql.resolver.OperatorResolution` before generation, and it - survives the generator's defensive tree copy. Direct callers that invoke - ``generate`` / ``giqlnearest_sql`` without running the pass (notably unit - tests) get the metadata resolved lazily here against this generator's - registered tables, so the emit path always reads resolved metadata. + survives the generator's defensive tree copy. The emitter reads only the + attached metadata; resolution lives entirely in the pass. :param expression: GIQLNearest expression node @@ -982,9 +914,6 @@ def _nearest_resolution(self, expression: GIQLNearest) -> OperatorResolution | N if resolution did not produce one. """ resolution = expression.meta.get(META_KEY) - if not isinstance(resolution, OperatorResolution): - resolve_operator_refs(expression.root(), self.tables) - resolution = expression.meta.get(META_KEY) return resolution if isinstance(resolution, OperatorResolution) else None def _raise_nearest_reference_error( @@ -1145,106 +1074,3 @@ def _extract_bool_param(param_expr: exp.Expression | None) -> bool: return param_expr.this else: return str(param_expr).upper() in ("TRUE", "1", "YES") - - def _get_column_refs( - self, - column_ref: str, - table_name: str | None = None, - include_strand: bool = False, - ) -> tuple[str, str, str] | tuple[str, str, str, str]: - """Get physical column names for genomic data. - - :param column_ref: - Logical column reference (e.g., 'v.interval' or 'interval') - :param table_name: - Table name to look up schema (optional, overrides extraction from column_ref) - :param include_strand: - If True, return 4-tuple with strand column; otherwise return 3-tuple - :return: - Tuple of (chromosome_col, start_col, end_col) or - (chromosome_col, start_col, end_col, strand_col) if include_strand=True - """ - # Default column names - chrom_col = DEFAULT_CHROM_COL - start_col = DEFAULT_START_COL - end_col = DEFAULT_END_COL - strand_col = DEFAULT_STRAND_COL - - # Alias is kept verbatim for output formatting; table name is resolved - # separately to look up the Table config (alias != name in joins). - table_alias = column_ref.rsplit(".", 1)[0] if "." in column_ref else None - table_name = self._resolve_table_name(column_ref, table_name) - - # Try to get custom column names from table config - if table_name and self.tables: - table = self.tables.get(table_name) - if table: - chrom_col = table.chrom_col - start_col = table.start_col - end_col = table.end_col - if table.strand_col: - strand_col = table.strand_col - - # Format with table alias if present - if table_alias: - base_cols = ( - f'{table_alias}."{chrom_col}"', - f'{table_alias}."{start_col}"', - f'{table_alias}."{end_col}"', - ) - if include_strand: - return base_cols + (f'{table_alias}."{strand_col}"',) - return base_cols - else: - base_cols = ( - f'"{chrom_col}"', - f'"{start_col}"', - f'"{end_col}"', - ) - if include_strand: - return base_cols + (f'"{strand_col}"',) - return base_cols - - def _resolve_table_name(self, column_ref: str, table_name: str | None) -> str | None: - """Resolve the underlying table name for a column reference. - - Precedence: explicit ``table_name`` (caller-provided context) > alias - map (JOIN-side resolution via ``self._alias_to_table``) > - ``self._current_table`` (FROM-clause fallback for unmapped dotted - aliases). Undotted refs return ``None`` because their caller is - expected to pass ``_current_table`` as the explicit ``table_name`` - when relevant. - - :param column_ref: - Column reference, possibly aliased (e.g. ``a.interval``) - :param table_name: - Explicit table name; takes precedence if non-empty - :return: - ``table_name`` if non-empty; otherwise the alias mapping from - ``self._alias_to_table`` (with ``self._current_table`` as fallback) - if ``column_ref`` is dotted; otherwise None - """ - if table_name: - return table_name - if "." in column_ref: - alias = column_ref.rsplit(".", 1)[0] - return self._alias_to_table.get(alias, self._current_table) - return None - - def _resolve_table( - self, column_ref: str, table_name: str | None = None - ) -> Table | None: - """Resolve the Table config that backs a column reference. - - :param column_ref: - Column reference, possibly aliased (e.g. ``a.interval``) - :param table_name: - Explicit table name; if omitted, derived from ``column_ref`` alias - or falls back to ``self._current_table`` - :return: - The Table config if registered, otherwise None - """ - table_name = self._resolve_table_name(column_ref, table_name) - if table_name and self.tables: - return self.tables.get(table_name) - return None From 6aa0c484237e919588a7d813ff4b86f1146e5431 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 10:10:07 -0400 Subject: [PATCH 054/142] test: Route direct-generate tests through the transpile pipeline Repoint the operator tests that invoked the generator on a bare parsed AST so they run the full resolve-then-canonicalize pipeline before generation, matching transpile() and preserving the coverage the deleted legacy fallback used to provide. Expected SQL is unchanged in every case; only the production call switches. Tests that raise before any metadata read stay on direct generation. --- tests/generators/test_base.py | 144 +++++++++------------------ tests/test_distance_transpilation.py | 49 +++++---- tests/test_distance_udf.py | 70 ++++++------- tests/test_nearest_transpilation.py | 37 ++++--- 4 files changed, 130 insertions(+), 170 deletions(-) diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index d316157..10a2d9c 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -156,10 +156,8 @@ def test_select_sql_with_alias(self, tables_info): THEN Alias-to-table mapping includes the alias. """ sql = "SELECT * FROM variants AS v WHERE v.interval INTERSECTS 'chr1:1000-2000'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_info) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_info) expected = ( "SELECT * FROM variants AS v WHERE " @@ -190,10 +188,8 @@ def test_intersects_sql_with_literal(self): THEN SQL with chrom = 'chr1' AND start < 2000 AND end > 1000 is generated. """ sql = "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:1000-2000'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) expected = ( "SELECT * FROM variants WHERE " @@ -212,10 +208,11 @@ def test_intersects_sql_column_join(self, tables_with_two_tables): "SELECT * FROM features_a AS a CROSS JOIN features_b AS b " "WHERE a.interval INTERSECTS b.interval" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + # The full transpile pipeline rewrites a column-to-column INTERSECTS into a + # binned equi-join before the predicate emitter runs, so run passes 1 and 2 + # directly to exercise the predicate emitter's column-join branch. + output = _generate_through_passes(sql, tables_with_two_tables) expected = ( "SELECT * FROM features_a AS a CROSS JOIN features_b AS b WHERE " @@ -239,9 +236,7 @@ def test_intersects_sql_validity_property(self, chrom_num, start, length): end = start + length sql = f"SELECT * FROM variants WHERE interval INTERSECTS '{chrom}:{start}-{end}'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) # Verify we can parse the output SQL (proves it's syntactically valid) parsed = parse_one(output) @@ -254,10 +249,8 @@ def test_contains_sql_point_query(self): THEN SQL with start <= 1000 AND end > 1000 is generated. """ sql = "SELECT * FROM variants WHERE interval CONTAINS 'chr1:1500'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) expected = ( "SELECT * FROM variants WHERE " @@ -272,10 +265,8 @@ def test_contains_sql_range_query(self): THEN SQL with start <= 1000 AND end >= 2000 is generated. """ sql = "SELECT * FROM variants WHERE interval CONTAINS 'chr1:1500-2000'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) expected = ( "SELECT * FROM variants WHERE " @@ -294,10 +285,8 @@ def test_contains_sql_column_join(self, tables_with_two_tables): "SELECT * FROM features_a AS a CROSS JOIN features_b AS b " "WHERE a.interval CONTAINS b.interval" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) expected = ( "SELECT * FROM features_a AS a CROSS JOIN features_b AS b WHERE " @@ -321,9 +310,7 @@ def test_contains_sql_coordinate_validity_property(self, chrom_num, start, lengt end = start + length sql = f"SELECT * FROM variants WHERE interval CONTAINS '{chrom}:{start}-{end}'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) # Verify we can parse the output SQL (proves it's syntactically valid) parsed = parse_one(output) @@ -339,10 +326,8 @@ def test_within_sql_with_literal(self): THEN SQL with start >= 1000 AND end <= 2000 is generated. """ sql = "SELECT * FROM variants WHERE interval WITHIN 'chr1:1000-5000'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) expected = ( "SELECT * FROM variants WHERE " @@ -360,10 +345,8 @@ def test_within_sql_column_join(self, tables_with_two_tables): "SELECT * FROM features_a AS a CROSS JOIN features_b AS b " "WHERE a.interval WITHIN b.interval" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) expected = ( "SELECT * FROM features_a AS a CROSS JOIN features_b AS b WHERE " @@ -382,10 +365,8 @@ def test_spatialsetpredicate_sql_any(self): "SELECT * FROM variants " "WHERE interval INTERSECTS ANY('chr1:1000-2000', 'chr1:5000-6000')" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) expected = ( "SELECT * FROM variants WHERE " @@ -404,10 +385,8 @@ def test_spatialsetpredicate_sql_all(self): "SELECT * FROM variants " "WHERE interval INTERSECTS ALL('chr1:1000-2000', 'chr1:1500-1800')" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) expected = ( "SELECT * FROM variants WHERE " @@ -423,10 +402,8 @@ def test_giqlnearest_sql_standalone(self, tables_with_peaks_and_genes): THEN Subquery with ORDER BY distance LIMIT k is generated. """ sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) expected = ( "SELECT * FROM (\n" @@ -459,10 +436,8 @@ def test_giqlnearest_sql_correlated(self, tables_with_peaks_and_genes): "SELECT * FROM peaks " "CROSS JOIN LATERAL NEAREST(genes, reference := peaks.interval, k := 3)" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) expected = ( "SELECT * FROM peaks CROSS JOIN LATERAL (\n" @@ -498,10 +473,8 @@ def test_giqlnearest_sql_with_max_distance(self, tables_with_peaks_and_genes): "CROSS JOIN LATERAL NEAREST(" "genes, reference := peaks.interval, k := 5, max_distance := 100000)" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) expected = ( "SELECT * FROM peaks CROSS JOIN LATERAL (\n" @@ -544,10 +517,8 @@ def test_giqlnearest_sql_stranded(self, tables_with_peaks_and_genes): "CROSS JOIN LATERAL NEAREST(" "genes, reference := peaks.interval, k := 3, stranded := true)" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) expected = ( "SELECT * FROM peaks CROSS JOIN LATERAL (\n" @@ -603,10 +574,9 @@ def test_giqlnearest_sql_implicit_outer_without_strand_column(self): "SELECT * FROM nostr " "CROSS JOIN LATERAL NEAREST(genes, k := 1, stranded := true)" ) - ast = parse_one(sql, dialect=GIQLDialect) # Act - output = BaseGIQLGenerator(tables=tables).generate(ast) + output = _generate_through_passes(sql, tables) # Assert assert "strand" not in output @@ -623,10 +593,8 @@ def test_giqlnearest_sql_signed(self, tables_with_peaks_and_genes): "CROSS JOIN LATERAL NEAREST(" "genes, reference := peaks.interval, k := 3, signed := true)" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) expected = ( "SELECT * FROM peaks CROSS JOIN LATERAL (\n" @@ -665,6 +633,8 @@ class NoLateralGenerator(BaseGIQLGenerator): # Use query without explicit reference to trigger correlated mode sql = "SELECT * FROM peaks CROSS JOIN LATERAL NEAREST(genes, k := 3)" ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables_with_peaks_and_genes) + ast = canonicalize_coordinates(ast) generator = NoLateralGenerator(tables=tables_with_peaks_and_genes) @@ -688,10 +658,8 @@ def test_giqlnearest_sql_parameter_handling_property( f"SELECT * FROM NEAREST(" f"genes, reference := 'chr1:1000-2000', k := {k}, max_distance := {max_distance})" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) # k should appear in LIMIT assert f"LIMIT {k}" in output @@ -708,10 +676,8 @@ def test_giqldistance_sql_basic(self, tables_with_two_tables): "SELECT DISTANCE(a.interval, b.interval) as dist " "FROM features_a a CROSS JOIN features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) expected = ( 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' @@ -733,10 +699,8 @@ def test_giqldistance_sql_stranded(self, tables_with_two_tables): "SELECT DISTANCE(a.interval, b.interval, stranded := true) as dist " "FROM features_a a CROSS JOIN features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) expected = ( 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' @@ -766,10 +730,8 @@ def test_giqldistance_sql_signed(self, tables_with_two_tables): "SELECT DISTANCE(a.interval, b.interval, signed := true) as dist " "FROM features_a a CROSS JOIN features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) expected = ( 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' @@ -792,10 +754,8 @@ def test_giqldistance_sql_stranded_and_signed(self, tables_with_two_tables): "DISTANCE(a.interval, b.interval, stranded := true, signed := true) as dist " "FROM features_a a CROSS JOIN features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) expected = ( 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' @@ -915,10 +875,8 @@ def test_giqlnearest_sql_stranded_literal_with_strand( "SELECT * FROM NEAREST(" "genes, reference := 'chr1:1000-2000:+', k := 3, stranded := true)" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) # Should contain strand literal '+' and strand filtering assert "'+'" in output @@ -933,10 +891,8 @@ def test_giqlnearest_sql_stranded_implicit_reference( THEN Strand column is resolved from outer table and used. """ sql = "SELECT * FROM peaks CROSS JOIN LATERAL NEAREST(genes, k := 3, stranded := true)" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_peaks_and_genes) # Should have strand columns from both tables assert 'peaks."strand"' in output @@ -982,6 +938,8 @@ def test_giqldistance_sql_literal_first_arg_error(self, tables_with_two_tables): """ sql = "SELECT DISTANCE('chr1:1000-2000', b.interval) as dist FROM features_b b" ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables_with_two_tables) + ast = canonicalize_coordinates(ast) generator = BaseGIQLGenerator(tables=tables_with_two_tables) @@ -996,6 +954,8 @@ def test_giqldistance_sql_literal_second_arg_error(self, tables_with_two_tables) """ sql = "SELECT DISTANCE(a.interval, 'chr1:1000-2000') as dist FROM features_a a" ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables_with_two_tables) + ast = canonicalize_coordinates(ast) generator = BaseGIQLGenerator(tables=tables_with_two_tables) @@ -1016,6 +976,8 @@ def test_giqlnearest_sql_missing_outer_table_error( this=exp.Table(this=exp.Identifier(this="genes")), k=exp.Literal.number(3), ) + resolve_operator_refs(nearest, tables_with_peaks_and_genes) + canonicalize_coordinates(nearest) generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) @@ -1034,12 +996,9 @@ def test_giqlnearest_sql_outer_table_not_in_tables(self): # The outer relation (unknown_table) is present in the LATERAL context # but is not a registered table, so the implicit-outer reference defers. sql = "SELECT * FROM unknown_table CROSS JOIN LATERAL NEAREST(genes, k := 3)" - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator(tables=tables) with pytest.raises(ValueError, match="not found in tables"): - generator.generate(ast) + _generate_through_passes(sql, tables) def test_giqlnearest_sql_invalid_reference_range(self, tables_with_peaks_and_genes): """ @@ -1048,12 +1007,9 @@ def test_giqlnearest_sql_invalid_reference_range(self, tables_with_peaks_and_gen THEN ValueError is raised with parse error details. """ sql = "SELECT * FROM NEAREST(genes, reference := 'invalid_range', k := 3)" - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) with pytest.raises(ValueError, match="Could not parse reference genomic range"): - generator.generate(ast) + _generate_through_passes(sql, tables_with_peaks_and_genes) def test_giqlnearest_sql_no_tables_error(self): """ @@ -1093,10 +1049,8 @@ def test_intersects_sql_unqualified_column(self): THEN Default column names are used without table qualifier. """ sql = "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:1000-2000'" - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate_through_passes(sql, Tables()) expected = ( "SELECT * FROM variants WHERE " @@ -1121,6 +1075,8 @@ def test_giqlnearest_sql_stranded_unqualified_reference( k=exp.Literal.number(3), stranded=exp.Boolean(this=True), ) + resolve_operator_refs(nearest, tables_with_peaks_and_genes) + canonicalize_coordinates(nearest) generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) output = generator.giqlnearest_sql(nearest) @@ -1144,6 +1100,8 @@ def test_giqlnearest_sql_identifier_target(self, tables_with_peaks_and_genes): reference=exp.Literal.string("chr1:1000-2000"), k=exp.Literal.number(3), ) + resolve_operator_refs(nearest, tables_with_peaks_and_genes) + canonicalize_coordinates(nearest) generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) output = generator.giqlnearest_sql(nearest) @@ -1168,10 +1126,8 @@ def test_giqldistance_stranded_param_truthy_values_property( f"SELECT DISTANCE(a.interval, b.interval, stranded := {bool_repr}) as dist " "FROM features_a a, features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) # Should include strand handling (NULL checks for strand columns) assert "strand" in output.lower() @@ -1193,10 +1149,8 @@ def test_giqldistance_stranded_param_falsy_values_property( f"SELECT DISTANCE(a.interval, b.interval, stranded := {bool_repr}) as dist " "FROM features_a a, features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) # Should NOT include strand NULL checks (basic distance) assert "strand" not in output.lower() @@ -1217,10 +1171,8 @@ def test_giqldistance_signed_param_truthy_values_property( f"SELECT DISTANCE(a.interval, b.interval, signed := {bool_repr}) as dist " "FROM features_a a, features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) # Signed distance has negative sign for upstream intervals assert "-(" in output @@ -1241,10 +1193,8 @@ def test_giqldistance_signed_param_falsy_values_property( f"SELECT DISTANCE(a.interval, b.interval, signed := {bool_repr}) as dist " "FROM features_a a, features_b b" ) - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) + output = _generate_through_passes(sql, tables_with_two_tables) # Unsigned distance has no negative sign (both ELSE branches are positive) # Count occurrences of "-(" - signed has 1, unsigned has 0 diff --git a/tests/test_distance_transpilation.py b/tests/test_distance_transpilation.py index 380aa19..08efe2a 100644 --- a/tests/test_distance_transpilation.py +++ b/tests/test_distance_transpilation.py @@ -6,8 +6,27 @@ from sqlglot import parse_one from giql import transpile +from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.generators import BaseGIQLGenerator +from giql.resolver import resolve_operator_refs +from giql.table import Tables + + +def _generate(sql: str, tables: Tables | None = None) -> str: + """Parse, run normalization passes 1 and 2, then generate SQL. + + DISTANCE operand resolution and coordinate canonicalization moved out of the + emitter and into the ResolveOperatorRefs / CanonicalizeCoordinates passes + (epic #114, issues #119 / #123). Emitter-level tests must run both passes + before generating, exactly as :func:`giql.transpile.transpile` does, rather + than calling ``generate`` on a bare parsed AST. + """ + tables = tables or Tables() + ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables) + ast = canonicalize_coordinates(ast) + return BaseGIQLGenerator(tables=tables).generate(ast) class TestDistanceTranspilation: @@ -24,9 +43,7 @@ def test_distance_transpilation_duckdb(self): FROM features_a a CROSS JOIN features_b b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate(sql) expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end") ELSE (a."start" - b."end") END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" @@ -43,9 +60,7 @@ def test_distance_transpilation_sqlite(self): FROM features_a a, features_b b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate(sql) expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end") ELSE (a."start" - b."end") END AS dist FROM features_a AS a, features_b AS b""" @@ -62,9 +77,7 @@ def test_distance_transpilation_postgres(self): FROM features_a a CROSS JOIN features_b b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate(sql) expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end") ELSE (a."start" - b."end") END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" @@ -72,11 +85,12 @@ def test_distance_transpilation_postgres(self): def test_distance_resolver_path_matches_direct_generation(self): """ - GIVEN a DISTANCE query over registered default-convention tables - WHEN transpiling through the full pipeline (the resolver pass) versus - generating directly from the parsed AST - THEN both paths should emit byte-identical SQL, proving the - ResolvedColumn metadata path reproduces the legacy string path + GIVEN a DISTANCE query over default-convention tables + WHEN transpiling through the full pipeline (with the tables registered) + versus running the two normalization passes with no tables registered + THEN both paths should emit byte-identical SQL, proving the qualified + ResolvedColumn metadata resolves identically whether or not the + operand's relation is registered """ query = ( "SELECT DISTANCE(a.interval, b.interval) AS dist " @@ -84,8 +98,7 @@ def test_distance_resolver_path_matches_direct_generation(self): ) via_transpile = transpile(query, tables=["features_a", "features_b"]) - ast = parse_one(query, dialect=GIQLDialect) - via_generate = BaseGIQLGenerator().generate(ast) + via_generate = _generate(query) assert via_transpile == via_generate @@ -101,9 +114,7 @@ def test_distance_transpilation_signed_duckdb(self): FROM features_a a CROSS JOIN features_b b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output = generator.generate(ast) + output = _generate(sql) # Signed distance: upstream (B before A) returns negative, # downstream (B after A) returns positive diff --git a/tests/test_distance_udf.py b/tests/test_distance_udf.py index c4c8535..8fa03d8 100644 --- a/tests/test_distance_udf.py +++ b/tests/test_distance_udf.py @@ -8,8 +8,26 @@ import pytest from sqlglot import parse_one +from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.generators import BaseGIQLGenerator +from giql.resolver import resolve_operator_refs +from giql.table import Tables + + +def _generate(sql: str) -> str: + """Parse, run normalization passes 1 and 2, then generate SQL. + + DISTANCE operand resolution and coordinate canonicalization moved out of the + emitter and into the ResolveOperatorRefs / CanonicalizeCoordinates passes + (epic #114, issues #119 / #123). These behavioral tests must run both passes + before generating, exactly as :func:`giql.transpile.transpile` does, rather + than calling ``generate`` on a bare parsed AST. + """ + ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, Tables()) + ast = canonicalize_coordinates(ast) + return BaseGIQLGenerator().generate(ast) class TestDistanceCalculation: @@ -32,9 +50,7 @@ def test_overlapping_intervals_return_zero(self): """ # Parse and generate SQL - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) # Verify SQL contains CASE expression logic for overlaps assert "CASE" in output_sql @@ -69,9 +85,7 @@ def test_non_overlapping_intervals_return_positive_distance(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -96,9 +110,7 @@ def test_different_chromosomes_return_null(self): (SELECT 'chr2' as chrom, 150 as start, 250 as end) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -127,9 +139,7 @@ def test_adjacent_bookended_intervals_return_zero(self): (SELECT 'chr1' as chrom, 200 as start, 300 as end) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -159,9 +169,7 @@ def test_zero_width_intervals_point_features(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -190,9 +198,7 @@ def test_stranded_same_strand_plus(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -217,9 +223,7 @@ def test_stranded_same_strand_minus(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '-' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -244,9 +248,7 @@ def test_stranded_different_strands_calculates_distance(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '-' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -271,9 +273,7 @@ def test_stranded_different_strands_minus_first(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -298,9 +298,7 @@ def test_stranded_dot_strand_returns_null(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '.' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -325,9 +323,7 @@ def test_stranded_question_mark_strand_returns_null(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -352,9 +348,7 @@ def test_stranded_null_strand_returns_null(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() @@ -379,9 +373,7 @@ def test_stranded_overlapping_intervals_minus_strand(self): (SELECT 'chr1' as chrom, 150 as start, 250 as end, '-' as strand) b """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator() - output_sql = generator.generate(ast) + output_sql = _generate(sql) conn = duckdb.connect(":memory:") result = conn.execute(output_sql).fetchone() diff --git a/tests/test_nearest_transpilation.py b/tests/test_nearest_transpilation.py index 7357707..2488cb0 100644 --- a/tests/test_nearest_transpilation.py +++ b/tests/test_nearest_transpilation.py @@ -8,11 +8,28 @@ from sqlglot import parse_one from giql import Table +from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.generators import BaseGIQLGenerator +from giql.resolver import resolve_operator_refs from giql.table import Tables +def _generate(sql: str, tables: Tables) -> str: + """Parse, run normalization passes 1 and 2, then generate SQL. + + Operator resolution and coordinate canonicalization moved out of the emitter + and into the ResolveOperatorRefs / CanonicalizeCoordinates passes (epic #114, + issues #118-#123). Emitter-level tests must run both passes before generating, + exactly as :func:`giql.transpile.transpile` does, rather than calling + ``generate`` on a bare parsed AST. + """ + ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables) + ast = canonicalize_coordinates(ast) + return BaseGIQLGenerator(tables=tables).generate(ast) + + @pytest.fixture def tables_with_peaks_and_genes(): """Tables container with peaks and genes tables.""" @@ -37,9 +54,7 @@ def test_nearest_basic_k3(self, tables_with_peaks_and_genes): CROSS JOIN LATERAL NEAREST(genes, reference := peaks.interval, k := 3) """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate(sql, tables_with_peaks_and_genes) # Expectations: # - LATERAL subquery @@ -65,9 +80,7 @@ def test_nearest_with_max_distance(self, tables_with_peaks_and_genes): CROSS JOIN LATERAL NEAREST(genes, reference := peaks.interval, k := 5, max_distance := 100000) """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate(sql, tables_with_peaks_and_genes) # Expectations: # - LATERAL subquery @@ -88,9 +101,7 @@ def test_nearest_standalone_literal(self, tables_with_peaks_and_genes): FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3) """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate(sql, tables_with_peaks_and_genes) # Expectations: # - No LATERAL (standalone mode) @@ -113,9 +124,7 @@ def test_nearest_with_stranded(self, tables_with_peaks_and_genes): CROSS JOIN LATERAL NEAREST(genes, reference := peaks.interval, k := 3, stranded := true) """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate(sql, tables_with_peaks_and_genes) # Expectations: # - LATERAL subquery @@ -138,9 +147,7 @@ def test_nearest_with_signed(self, tables_with_peaks_and_genes): CROSS JOIN LATERAL NEAREST(genes, reference := peaks.interval, k := 3, signed := true) """ - ast = parse_one(sql, dialect=GIQLDialect) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.generate(ast) + output = _generate(sql, tables_with_peaks_and_genes) # Expectations: # - LATERAL subquery From ccf45a270e40601bc0f104c4c2f48336d2466e02 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 11:02:52 -0400 Subject: [PATCH 055/142] feat: Add optional predicate to CLUSTER and MERGE CLUSTER and MERGE accept an optional named predicate argument that further restricts which adjacent intervals are coalesced. The cluster-boundary condition becomes adjacent AND predicate: an interval stays in the current cluster only when it is within distance of its predecessor and the predicate holds between the two. An equality predicate therefore yields a run-length encoding of the input. Bare columns in the predicate resolve to the current interval; the predecessor's value is referenced with a prev. qualifier, rewritten to a LAG window over the operator's existing partition and order. MERGE inherits predicate-aware boundaries by passing the predicate through to its underlying CLUSTER. Omitting the predicate preserves the prior adjacency-only behavior exactly. --- src/giql/expressions.py | 17 ++++++++++ src/giql/transformer.py | 69 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index e8de483..5143d31 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -199,17 +199,25 @@ class GIQLCluster(exp.Func): Implicitly partitions by chromosome and orders by start position. + The optional ``predicate`` argument is a boolean expression evaluated + between each interval and its sorted predecessor; intervals are only kept + in the same cluster when they are adjacent *and* the predicate holds. Bare + columns resolve to the current interval; the predecessor's value of a + column is referenced with a ``prev.*`` qualifier. + Examples: CLUSTER(interval) CLUSTER(interval, 1000) CLUSTER(interval, stranded := true) CLUSTER(interval, 1000, stranded := true) + CLUSTER(interval, predicate := depth = prev.depth) """ arg_types = { "this": True, # genomic column "distance": False, # maximum distance between features "stranded": False, # strand-specific clustering + "predicate": False, # pairwise boolean gate (current row vs prev.*) } @classmethod @@ -232,16 +240,25 @@ class GIQLMerge(exp.Func): Merges overlapping or bookended intervals into single intervals. Built on top of CLUSTER operation. + The optional ``predicate`` argument gates merging on a pairwise boolean + expression between each interval and its sorted predecessor (see + :class:`GIQLCluster`); a ``prev.*`` qualifier references the predecessor's + value of a column. When the predicate tests equality of a value this + yields a run-length encoding of the input interval sequence. + Examples: MERGE(interval) MERGE(interval, 1000) MERGE(interval, stranded := true) + MERGE(interval, predicate := depth = prev.depth) + MERGE(interval, predicate := strand = prev.strand AND name = prev.name) """ arg_types = { "this": True, # genomic column "distance": False, # maximum distance between features "stranded": False, # strand-specific merging + "predicate": False, # pairwise boolean gate (current row vs prev.*) } @classmethod diff --git a/src/giql/transformer.py b/src/giql/transformer.py index b0355c6..66d64b2 100644 --- a/src/giql/transformer.py +++ b/src/giql/transformer.py @@ -443,14 +443,34 @@ def _transform_for_cluster( else: lag_with_distance = lag_window + # Build the adjacency condition (predecessor end >= current start). + adjacency = exp.GTE( + this=lag_with_distance, + expression=exp.column(start_col, quoted=True), + ) + + # An optional predicate further restricts which adjacent intervals + # are kept together: a row stays in the current cluster only when it + # is adjacent to its predecessor AND the predicate holds between them. + # ``prev.col`` references in the predicate resolve to the predecessor + # row via LAG over the same partition/order as the adjacency window. + predicate_expr = cluster_expr.args.get("predicate") + if predicate_expr is not None: + rewritten_predicate = self._rewrite_prev_refs( + predicate_expr, partition_cols, order_by + ) + keep_together = exp.And( + this=adjacency, + expression=exp.Paren(this=rewritten_predicate), + ) + else: + keep_together = adjacency + # Create CASE expression for is_new_cluster case_expr = exp.Case( ifs=[ exp.If( - this=exp.GTE( - this=lag_with_distance, - expression=exp.column(start_col, quoted=True), - ), + this=keep_together, true=exp.Literal.number(0), ) ], @@ -550,6 +570,44 @@ def _transform_for_cluster( return new_query + def _rewrite_prev_refs( + self, + predicate: exp.Expression, + partition_cols: list[exp.Expression], + order_by: list[exp.Ordered], + ) -> exp.Expression: + """Rewrite ``prev.col`` references in a predicate to LAG windows. + + Bare column references in the predicate denote the current interval and + are left untouched. Each ``prev.col`` reference denotes the sorted + predecessor and is rewritten to ``LAG(col) OVER (...)`` using the same + partition/order as the cluster's adjacency window, so the predicate is + evaluated pairwise against the immediately preceding row. + + :param predicate: + Boolean predicate expression to rewrite (not mutated). + :param partition_cols: + Window partition columns (chromosome, optionally strand). + :param order_by: + Window ORDER BY terms (start position). + :return: + A copy of the predicate with every ``prev.*`` reference replaced by + an equivalent LAG window. + """ + + def _replace(node: exp.Expression) -> exp.Expression: + if isinstance(node, exp.Column) and node.table == "prev": + return exp.Window( + this=exp.Anonymous( + this="LAG", expressions=[exp.Column(this=node.this.copy())] + ), + partition_by=[col.copy() for col in partition_cols], + order=exp.Order(expressions=[term.copy() for term in order_by]), + ) + return node + + return predicate.copy().transform(_replace) + class MergeTransformer: """Transforms queries containing MERGE into GROUP BY queries. @@ -673,6 +731,7 @@ def _transform_for_merge( # Extract MERGE parameters (same as CLUSTER) distance_expr = merge_expr.args.get("distance") stranded_expr = merge_expr.args.get("stranded") + predicate_expr = merge_expr.args.get("predicate") # Get column names from table config or use defaults ( @@ -688,6 +747,8 @@ def _transform_for_merge( cluster_kwargs["distance"] = distance_expr if stranded_expr: cluster_kwargs["stranded"] = stranded_expr + if predicate_expr is not None: + cluster_kwargs["predicate"] = predicate_expr cluster_expr = GIQLCluster(**cluster_kwargs) From 2f3e78e973d84a3ae4d22ce4aea995dc55aecfcb Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 11:03:04 -0400 Subject: [PATCH 056/142] test: Cover CLUSTER and MERGE predicate behavior Add parsing tests for the predicate argument under both the := and => named-argument forms and the prev. qualifier; transpilation-shape tests asserting the predicate ANDs into the cluster-boundary CASE, that prev. references become LAG windows over the operator's partition and order, and that the predicate-free path stays byte-identical. Add DuckDB-executed functional tests in a new bedtools-free integration package: run-length encoding on a column, single-linkage drift for a non-equivalence predicate, distance and stranded composition, per-chromosome partition reset, and the DISJOIN to depth-aggregation to predicate-MERGE pipeline that reconstructs disjoin()-style output. --- .../integration/cluster_predicate/__init__.py | 0 .../integration/cluster_predicate/conftest.py | 47 +++ .../test_cluster_predicate.py | 364 ++++++++++++++++++ tests/test_cluster_parsing.py | 96 +++++ tests/test_cluster_predicate_transpilation.py | 148 +++++++ tests/test_merge_parsing.py | 72 +++- 6 files changed, 724 insertions(+), 3 deletions(-) create mode 100644 tests/integration/cluster_predicate/__init__.py create mode 100644 tests/integration/cluster_predicate/conftest.py create mode 100644 tests/integration/cluster_predicate/test_cluster_predicate.py create mode 100644 tests/test_cluster_predicate_transpilation.py diff --git a/tests/integration/cluster_predicate/__init__.py b/tests/integration/cluster_predicate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/cluster_predicate/conftest.py b/tests/integration/cluster_predicate/conftest.py new file mode 100644 index 0000000..c6a00e3 --- /dev/null +++ b/tests/integration/cluster_predicate/conftest.py @@ -0,0 +1,47 @@ +"""Pytest fixtures for CLUSTER/MERGE predicate integration tests. + +These tests assert the functional behavior of the optional ``predicate :=`` +argument by executing the generated SQL against DuckDB. They do not invoke +``bedtools`` or ``pybedtools`` -- the comparison reference is a partition +computed directly in the test, matching the DISJOIN coordinate-space approach. +""" + +import pytest + +from giql import transpile + +duckdb = pytest.importorskip("duckdb") + +pytestmark = pytest.mark.integration + +from tests.integration.bedtools.utils.duckdb_loader import load_intervals # noqa: E402 + + +@pytest.fixture(scope="function") +def duckdb_connection(): + """Provide a clean DuckDB connection for each test.""" + conn = duckdb.connect(":memory:") + yield conn + conn.close() + + +@pytest.fixture(scope="function") +def giql_query(duckdb_connection): + """Provide a helper that loads data, transpiles GIQL, and executes. + + Usage:: + + rows = giql_query( + "SELECT MERGE(interval, predicate := score = prev.score) FROM t", + tables=["t"], + t=[("chr1", 0, 10, "a", 5, "+"), ...], + ) + """ + + def _run(query: str, *, tables: list[str], **table_data): + for name, intervals in table_data.items(): + load_intervals(duckdb_connection, name, intervals) + sql = transpile(query, tables=tables) + return duckdb_connection.execute(sql).fetchall() + + return _run diff --git a/tests/integration/cluster_predicate/test_cluster_predicate.py b/tests/integration/cluster_predicate/test_cluster_predicate.py new file mode 100644 index 0000000..dead147 --- /dev/null +++ b/tests/integration/cluster_predicate/test_cluster_predicate.py @@ -0,0 +1,364 @@ +"""Integration tests for the CLUSTER/MERGE predicate argument. + +These execute the generated SQL against DuckDB and assert the functional +behavior of ``predicate :=``: run-length encoding of equal-valued runs, +reconstruction of Bioconductor ``disjoin()``-style depth-annotated output via a +DISJOIN -> depth-aggregation -> predicate-MERGE pipeline, and the documented +single-linkage drift for non-equivalence predicates. +""" + +import pytest + +pytestmark = pytest.mark.integration + + +def _ival(chrom, start, end, name, score, strand="+"): + """Build a 6-tuple for ``load_intervals``.""" + return (chrom, start, end, name, score, strand) + + +class TestMergePredicate: + """Functional behavior of MERGE(interval, predicate := ...).""" + + def test_merge_without_predicate_coalesces_adjacent_run(self, giql_query): + """Test that MERGE without a predicate collapses an adjacent run. + + Given: + Five abutting intervals spanning [0, 50) with mixed scores. + When: + MERGE(interval) runs with no predicate. + Then: + It should collapse the whole adjacent run into a single interval. + """ + # Arrange & act + rows = giql_query( + "SELECT MERGE(interval) FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 5), + _ival("chr1", 10, 20, "b", 5), + _ival("chr1", 20, 30, "c", 3), + _ival("chr1", 30, 40, "d", 3), + _ival("chr1", 40, 50, "e", 5), + ], + ) + + # Assert + assert rows == [("chr1", 0, 50)] + + def test_merge_with_equality_predicate_run_length_encodes(self, giql_query): + """Test that an equality predicate run-length-encodes adjacent runs. + + Given: + Five abutting intervals whose scores form the runs 5,5 | 3,3 | 5. + When: + MERGE(interval, predicate := score = prev.score) runs. + Then: + It should emit one merged interval per maximal equal-score run. + """ + # Arrange & act + rows = giql_query( + "SELECT MERGE(interval, predicate := score = prev.score) FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 5), + _ival("chr1", 10, 20, "b", 5), + _ival("chr1", 20, 30, "c", 3), + _ival("chr1", 30, 40, "d", 3), + _ival("chr1", 40, 50, "e", 5), + ], + ) + + # Assert + assert rows == [ + ("chr1", 0, 20), + ("chr1", 20, 40), + ("chr1", 40, 50), + ] + + def test_merge_predicate_drifts_under_single_linkage(self, giql_query): + """Test that a non-equivalence predicate exhibits single-linkage drift. + + Given: + Three abutting intervals with scores 10, 13, 16, where each + consecutive pair differs by 3 but the extremes differ by 6. + When: + MERGE(interval, predicate := ABS(score - prev.score) < 5) runs. + Then: + It should merge the entire run into one interval even though the + cluster's extremes violate the predicate (documented drift). + """ + # Arrange & act + rows = giql_query( + "SELECT MERGE(interval, predicate := ABS(score - prev.score) < 5) " + "FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 10), + _ival("chr1", 10, 20, "b", 13), + _ival("chr1", 20, 30, "c", 16), + ], + ) + + # Assert + assert rows == [("chr1", 0, 30)] + + def test_merge_compound_predicate_breaks_on_either_column(self, giql_query): + """Test that a multi-column AND predicate breaks on any column change. + + Given: + Four abutting intervals where the (name, strand) pair changes first + on strand and then on name partway through the run. + When: + MERGE(interval, predicate := strand = prev.strand AND + name = prev.name) runs. + Then: + It should break the merge wherever either column changes, yielding + one merged interval per homogeneous (name, strand) run. + """ + # Arrange & act + rows = giql_query( + "SELECT MERGE(interval, predicate := strand = prev.strand " + "AND name = prev.name) FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "g", 5, "+"), + _ival("chr1", 10, 20, "g", 5, "+"), + _ival("chr1", 20, 30, "g", 5, "-"), + _ival("chr1", 30, 40, "h", 5, "-"), + ], + ) + + # Assert + assert rows == [ + ("chr1", 0, 20), + ("chr1", 20, 30), + ("chr1", 30, 40), + ] + + def test_merge_stranded_predicate_evaluates_within_strand(self, giql_query): + """Test that the predicate is evaluated within each strand partition. + + Given: + Equal-name abutting intervals on both the + and - strands. + When: + MERGE(interval, stranded := true, predicate := name = prev.name) + runs. + Then: + It should merge each strand's equal-name run independently, emitting + one interval per strand rather than collapsing across strands. + """ + # Arrange & act + rows = giql_query( + "SELECT MERGE(interval, stranded := true, predicate := name = prev.name) " + "FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 5, "+"), + _ival("chr1", 10, 20, "a", 5, "+"), + _ival("chr1", 0, 10, "a", 5, "-"), + _ival("chr1", 10, 20, "a", 5, "-"), + ], + ) + + # Assert + assert rows == [ + ("chr1", "+", 0, 20), + ("chr1", "-", 0, 20), + ] + + +class TestClusterPredicate: + """Functional behavior of CLUSTER(interval, predicate := ...).""" + + def test_cluster_with_equality_predicate_assigns_run_ids(self, giql_query): + """Test that an equality predicate assigns one cluster id per run. + + Given: + Five abutting intervals whose scores form the runs 5,5 | 3,3 | 5. + When: + CLUSTER(interval, predicate := score = prev.score) runs. + Then: + It should assign a distinct cluster id to each maximal equal-score + run, comparing each row only to its immediate predecessor. + """ + # Arrange & act + rows = giql_query( + "SELECT name, CLUSTER(interval, predicate := score = prev.score) AS cid " + "FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 5), + _ival("chr1", 10, 20, "b", 5), + _ival("chr1", 20, 30, "c", 3), + _ival("chr1", 30, 40, "d", 3), + _ival("chr1", 40, 50, "e", 5), + ], + ) + + # Assert + cid = dict(rows) + assert cid["a"] == cid["b"] + assert cid["c"] == cid["d"] + assert len({cid["a"], cid["c"], cid["e"]}) == 3 + + def test_cluster_distance_and_predicate_compose(self, giql_query): + """Test that the distance gate and predicate gate both apply. + + Given: + Three equal-score intervals separated by a 50bp gap then a 150bp + gap, clustered with CLUSTER(interval, 100, predicate := score = + prev.score). + When: + The generated SQL runs in DuckDB. + Then: + It should keep the 50bp-gap pair together but start a new cluster at + the 150bp gap even though the scores match, proving distance and + predicate compose. + """ + # Arrange & act + rows = giql_query( + "SELECT name, CLUSTER(interval, 100, predicate := score = prev.score) " + "AS cid FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 100, 200, "i1", 5), + _ival("chr1", 250, 350, "i2", 5), # 50bp gap, score matches + _ival("chr1", 500, 600, "i3", 5), # 150bp gap, score matches + ], + ) + + # Assert + cid = dict(rows) + assert cid["i1"] == cid["i2"] + assert cid["i3"] != cid["i1"] + + def test_cluster_predicate_resets_at_chromosome_boundary(self, giql_query): + """Test that the predicate does not carry across chromosomes. + + Given: + Equal-score abutting intervals on chr1 and chr2, clustered with + CLUSTER(interval, predicate := score = prev.score). + When: + The generated SQL runs in DuckDB. + Then: + It should begin a fresh cluster for the first row of each + chromosome, since the predecessor LAG resets at the per-chromosome + partition boundary. + """ + # Arrange & act + rows = giql_query( + "SELECT chrom, name, CLUSTER(interval, predicate := score = prev.score) " + "AS cid FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 5), + _ival("chr1", 10, 20, "b", 5), + _ival("chr2", 0, 10, "c", 5), + _ival("chr2", 10, 20, "d", 5), + ], + ) + + # Assert + by_name = {name: (chrom, cid) for chrom, name, cid in rows} + # Same chromosome, equal score, adjacent -> same cluster. + assert by_name["a"][1] == by_name["b"][1] + assert by_name["c"][1] == by_name["d"][1] + # chr2's first row starts its own cluster (ids are per-partition). + assert {by_name["a"], by_name["c"]} == {("chr1", 1), ("chr2", 1)} + + +class TestDisjoinDepthMergePipeline: + """DISJOIN -> depth-aggregation -> predicate-MERGE reconstructs disjoin().""" + + _FEATURES = [ + _ival("chr1", 0, 20, "a", 0), + _ival("chr1", 0, 20, "b", 0), + _ival("chr1", 20, 40, "c", 0), + _ival("chr1", 20, 40, "d", 0), + _ival("chr1", 10, 30, "e", 0), + ] + # Per-breakpoint coverage depth: [0,10)=2, [10,20)=3, [20,30)=3, [30,40)=2. + + _DEPTH_SEGMENTS_QUERY = """ + SELECT disjoin_chrom AS chrom, + disjoin_start AS start, + disjoin_end AS "end", + COUNT(*) AS depth + FROM DISJOIN(features) + GROUP BY disjoin_chrom, disjoin_start, disjoin_end + """ + + def test_disjoin_depth_segments_form_expected_partition(self, giql_query): + """Test that DISJOIN + depth aggregation yields per-breakpoint depths. + + Given: + Two doubled abutting regions overlaid by a spanning interval, + producing per-breakpoint coverage depths 2, 3, 3, 2. + When: + DISJOIN(features) is aggregated to per-segment coverage depth. + Then: + It should yield four disjoint segments carrying those depths. + """ + # Arrange & act + rows = giql_query( + self._DEPTH_SEGMENTS_QUERY + ' ORDER BY "start"', + tables=["features"], + features=self._FEATURES, + ) + + # Assert + assert rows == [ + ("chr1", 0, 10, 2), + ("chr1", 10, 20, 3), + ("chr1", 20, 30, 3), + ("chr1", 30, 40, 2), + ] + + def test_predicate_merge_remerges_equal_depth_segments(self, giql_query): + """Test that predicate-MERGE re-clusters runs of equal coverage depth. + + Given: + The depth-annotated disjoint segments (depths 2, 3, 3, 2). + When: + MERGE(interval, predicate := depth = prev.depth) runs over them. + Then: + It should coalesce the adjacent depth-3 run into [10, 30) while + keeping the depth-2 flanks distinct, reproducing the re-clustered + partition Bioconductor disjoin() emits. + """ + # Arrange & act + rows = giql_query( + "SELECT MERGE(interval, predicate := depth = prev.depth) " + f"FROM ({self._DEPTH_SEGMENTS_QUERY}) AS segments", + tables=["features"], + features=self._FEATURES, + ) + + # Assert + assert rows == [ + ("chr1", 0, 10), + ("chr1", 10, 30), + ("chr1", 30, 40), + ] + + def test_predicate_merge_differs_from_unconditioned_merge(self, giql_query): + """Test that the predicate is what preserves the depth boundaries. + + Given: + The same depth-annotated disjoint segments. + When: + MERGE(interval) runs over them with no predicate. + Then: + It should collapse every adjacent segment into one interval, + confirming the predicate alone preserves the coverage structure. + """ + # Arrange & act + rows = giql_query( + f"SELECT MERGE(interval) FROM ({self._DEPTH_SEGMENTS_QUERY}) AS segments", + tables=["features"], + features=self._FEATURES, + ) + + # Assert + assert rows == [("chr1", 0, 40)] diff --git a/tests/test_cluster_parsing.py b/tests/test_cluster_parsing.py index adfb116..0dd7ec6 100644 --- a/tests/test_cluster_parsing.py +++ b/tests/test_cluster_parsing.py @@ -6,6 +6,7 @@ """ import pytest +from sqlglot import exp from sqlglot import parse_one from sqlglot.errors import ParseError @@ -95,3 +96,98 @@ def test_from_arg_list_should_reject_missing_target(self): "SELECT CLUSTER(stranded := true) AS cluster_id FROM peaks", dialect=GIQLDialect, ) + + def test_from_arg_list_with_predicate(self): + """Test that a predicate named argument is captured on the node. + + Given: + A GIQL query with CLUSTER(interval, predicate := depth = prev.depth). + When: + Parsing the query. + Then: + It should attach the predicate as an equality expression in args. + """ + # Act + ast = parse_one( + "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "FROM peaks", + dialect=GIQLDialect, + ) + + # Assert + cluster_expr = ast.expressions[1].this + assert isinstance(cluster_expr, GIQLCluster) + assert isinstance(cluster_expr.args.get("predicate"), exp.EQ) + + def test_from_arg_list_with_predicate_prev_qualifier(self): + """Test that a prev. qualifier parses as a column on the predecessor. + + Given: + A predicate CLUSTER(interval, predicate := depth = prev.depth) + referencing the predecessor row with the prev. qualifier. + When: + Parsing the query. + Then: + It should parse prev.depth as a column whose table is prev. + """ + # Act + ast = parse_one( + "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "FROM peaks", + dialect=GIQLDialect, + ) + + # Assert + predicate = ast.expressions[1].this.args["predicate"] + prev_ref = predicate.expression + assert isinstance(prev_ref, exp.Column) + assert prev_ref.table == "prev" + assert prev_ref.name == "depth" + + def test_from_arg_list_with_predicate_kwarg_syntax(self): + """Test that the => kwarg form also binds the predicate argument. + + Given: + A CLUSTER call using CLUSTER(interval, predicate => score = prev.score) + with the => kwarg form rather than :=. + When: + Parsing the query. + Then: + It should attach the predicate as an equality expression in args. + """ + # Act + ast = parse_one( + "SELECT *, CLUSTER(interval, predicate => score = prev.score) AS cid " + "FROM peaks", + dialect=GIQLDialect, + ) + + # Assert + cluster_expr = ast.expressions[1].this + assert isinstance(cluster_expr, GIQLCluster) + assert isinstance(cluster_expr.args.get("predicate"), exp.EQ) + + def test_from_arg_list_with_predicate_and_positional_distance(self): + """Test that a predicate composes with positional distance and stranded. + + Given: + A query mixing a positional distance, stranded :=, and predicate := + on a single CLUSTER call. + When: + Parsing the query. + Then: + It should retain distance, stranded, and predicate together in args. + """ + # Act + ast = parse_one( + "SELECT *, CLUSTER(interval, 1000, stranded := true, " + "predicate := name = prev.name) AS cid FROM peaks", + dialect=GIQLDialect, + ) + + # Assert + cluster_expr = ast.expressions[1].this + assert isinstance(cluster_expr, GIQLCluster) + assert cluster_expr.args.get("distance") is not None + assert cluster_expr.args.get("stranded") is not None + assert cluster_expr.args.get("predicate") is not None diff --git a/tests/test_cluster_predicate_transpilation.py b/tests/test_cluster_predicate_transpilation.py new file mode 100644 index 0000000..f553543 --- /dev/null +++ b/tests/test_cluster_predicate_transpilation.py @@ -0,0 +1,148 @@ +"""Transpilation tests for the CLUSTER/MERGE predicate argument. + +Tests verify that an optional ``predicate :=`` argument ANDs into the +cluster-boundary CASE, that ``prev.col`` references are rewritten to LAG +windows over the operator's partition/order, and that omitting the predicate +leaves the emitted SQL byte-identical to the pre-predicate behavior. +""" + +from giql import transpile + + +class TestClusterPredicateTranspilation: + """Tests for CLUSTER predicate transpilation to SQL.""" + + def test_transpile_without_predicate_is_unchanged(self): + """Test that a predicate-free CLUSTER transpiles to the legacy shape. + + Given: + A CLUSTER query with no predicate argument. + When: + Transpiling to SQL. + Then: + It should emit the bare adjacency CASE with no AND clause. + """ + # Act + sql = transpile( + "SELECT *, CLUSTER(interval) AS cid FROM peaks", tables=["peaks"] + ) + + # Assert + assert ( + 'CASE WHEN LAG("end") OVER (PARTITION BY "chrom" ORDER BY "start" ' + 'NULLS LAST) >= "start" THEN 0 ELSE 1 END' in sql + ) + + def test_transpile_with_predicate_ands_into_case(self): + """Test that a predicate ANDs into the cluster-boundary CASE. + + Given: + A CLUSTER query with predicate := depth = prev.depth. + When: + Transpiling to SQL. + Then: + It should AND the predicate onto the adjacency test inside the CASE. + """ + # Act + sql = transpile( + "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "FROM peaks", + tables=["peaks"], + ) + + # Assert + assert ' >= "start" AND (' in sql + + def test_transpile_rewrites_prev_ref_to_lag_window(self): + """Test that a prev. reference becomes a LAG over the cluster window. + + Given: + A CLUSTER query with predicate := depth = prev.depth. + When: + Transpiling to SQL. + Then: + It should rewrite prev.depth to LAG(depth) over the same + partition/order as the adjacency window. + """ + # Act + sql = transpile( + "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "FROM peaks", + tables=["peaks"], + ) + + # Assert + assert ( + 'depth = LAG(depth) OVER (PARTITION BY "chrom" ORDER BY "start" ' + "NULLS LAST)" in sql + ) + + def test_transpile_predicate_uses_stranded_partition(self): + """Test that prev. LAG windows honor the stranded partition. + + Given: + A stranded CLUSTER query with a predicate referencing prev.name. + When: + Transpiling to SQL. + Then: + It should partition the predicate's LAG window by chrom and strand. + """ + # Act + sql = transpile( + "SELECT *, CLUSTER(interval, stranded := true, " + "predicate := name = prev.name) AS cid FROM peaks", + tables=["peaks"], + ) + + # Assert + assert ( + 'LAG(name) OVER (PARTITION BY "chrom", "strand" ORDER BY "start" ' + "NULLS LAST)" in sql + ) + + +class TestMergePredicateTranspilation: + """Tests for MERGE predicate transpilation to SQL.""" + + def test_transpile_without_predicate_is_unchanged(self): + """Test that a predicate-free MERGE transpiles to the legacy shape. + + Given: + A MERGE query with no predicate argument. + When: + Transpiling to SQL. + Then: + It should emit the bare adjacency CASE with no AND clause. + """ + # Act + sql = transpile("SELECT MERGE(interval) FROM peaks", tables=["peaks"]) + + # Assert + assert ( + 'CASE WHEN LAG("end") OVER (PARTITION BY "chrom" ORDER BY "start" ' + 'NULLS LAST) >= "start" THEN 0 ELSE 1 END' in sql + ) + + def test_transpile_predicate_inherited_through_cluster(self): + """Test that MERGE inherits predicate-aware cluster boundaries. + + Given: + A MERGE query with predicate := depth = prev.depth. + When: + Transpiling to SQL. + Then: + It should AND the rewritten predicate into the underlying CLUSTER + CASE that drives the GROUP BY. + """ + # Act + sql = transpile( + "SELECT MERGE(interval, predicate := depth = prev.depth) FROM peaks", + tables=["peaks"], + ) + + # Assert + assert ' >= "start" AND (' in sql + assert ( + 'depth = LAG(depth) OVER (PARTITION BY "chrom" ORDER BY "start" ' + "NULLS LAST)" in sql + ) diff --git a/tests/test_merge_parsing.py b/tests/test_merge_parsing.py index 6ae104c..eb94e3c 100644 --- a/tests/test_merge_parsing.py +++ b/tests/test_merge_parsing.py @@ -6,6 +6,7 @@ """ import pytest +from sqlglot import exp from sqlglot import parse_one from sqlglot.errors import ParseError @@ -84,6 +85,71 @@ def test_from_arg_list_should_reject_missing_target(self): """ # Arrange, act, & assert with pytest.raises(ParseError, match="requires a genomic interval"): - parse_one( - "SELECT MERGE(stranded := true) FROM peaks", dialect=GIQLDialect - ) + parse_one("SELECT MERGE(stranded := true) FROM peaks", dialect=GIQLDialect) + + def test_from_arg_list_with_predicate(self): + """Test that a predicate named argument is captured on the node. + + Given: + A GIQL query with MERGE(interval, predicate := depth = prev.depth). + When: + Parsing the query. + Then: + It should attach the predicate as an equality expression in args. + """ + # Act + ast = parse_one( + "SELECT MERGE(interval, predicate := depth = prev.depth) FROM peaks", + dialect=GIQLDialect, + ) + + # Assert + merge_expr = ast.expressions[0] + assert isinstance(merge_expr, GIQLMerge) + assert isinstance(merge_expr.args.get("predicate"), exp.EQ) + + def test_from_arg_list_with_predicate_kwarg_syntax(self): + """Test that the => kwarg form also binds the predicate argument. + + Given: + A MERGE call using MERGE(interval, predicate => score = prev.score) + with the => kwarg form rather than :=. + When: + Parsing the query. + Then: + It should attach the predicate as an equality expression in args. + """ + # Act + ast = parse_one( + "SELECT MERGE(interval, predicate => score = prev.score) FROM peaks", + dialect=GIQLDialect, + ) + + # Assert + merge_expr = ast.expressions[0] + assert isinstance(merge_expr, GIQLMerge) + assert isinstance(merge_expr.args.get("predicate"), exp.EQ) + + def test_from_arg_list_with_compound_predicate(self): + """Test that a multi-term predicate parses as a conjunction. + + Given: + A query MERGE(interval, predicate := strand = prev.strand AND + name = prev.name) joining two pairwise comparisons with AND. + When: + Parsing the query. + Then: + It should attach the predicate as an AND of two prev. comparisons. + """ + # Act + ast = parse_one( + "SELECT MERGE(interval, predicate := strand = prev.strand " + "AND name = prev.name) FROM peaks", + dialect=GIQLDialect, + ) + + # Assert + predicate = ast.expressions[0].args["predicate"] + assert isinstance(predicate, exp.And) + prev_tables = {col.table for col in predicate.find_all(exp.Column)} + assert "prev" in prev_tables From 92e2e3265fb651d7490c0288d6724144e032f994 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 11:03:13 -0400 Subject: [PATCH 057/142] docs: Document the CLUSTER and MERGE predicate argument Document the predicate argument in the aggregation-operator reference and the clustering recipe: the prev. qualifier convention, composition with distance and stranded, the references-existing-columns-only constraint, and the pairwise-only single-linkage caveat. Add recipes for run-length encoding and the disjoin() coverage-segment reconstruction. Surface the argument in the MCP operator catalog. --- docs/dialect/aggregation-operators.rst | 77 +++++++++++++++++++++++++ docs/recipes/clustering.rst | 80 ++++++++++++++++++++++++++ src/giql/mcp/server.py | 16 ++++++ 3 files changed, 173 insertions(+) diff --git a/docs/dialect/aggregation-operators.rst b/docs/dialect/aggregation-operators.rst index 9887b87..5c9893d 100644 --- a/docs/dialect/aggregation-operators.rst +++ b/docs/dialect/aggregation-operators.rst @@ -40,6 +40,9 @@ Syntax -- Strand-specific clustering CLUSTER(interval, stranded := true) AS cluster_id + -- Predicate-gated clustering (run-length encoding on a column) + CLUSTER(interval, predicate := depth = prev.depth) AS cluster_id + -- Combined parameters CLUSTER(interval, distance, stranded := true) AS cluster_id @@ -56,6 +59,36 @@ Parameters **stranded** *(optional)* When ``true``, only cluster intervals on the same strand. Default: ``false``. +**predicate** *(optional)* + A boolean expression evaluated between each interval and its sorted + predecessor. When supplied, the cluster-boundary condition becomes + **adjacent AND predicate**: an interval stays in the current cluster only + when it is within ``distance`` of its predecessor *and* the predicate holds + between the two. A change in the predicate forces a new cluster, so an + equality predicate yields a run-length encoding of the input sequence. + Omitting the predicate preserves the default adjacency-only behavior. + + Bare column references resolve to the *current* interval; the predecessor's + value of a column is referenced with a ``prev.*`` qualifier + (e.g. ``depth = prev.depth``). The predicate composes with ``distance`` and + ``stranded`` and is evaluated under the operator's existing per-chromosome + (and per-strand) partition and start-position order. + + Two constraints apply: + + - **References existing columns only.** The predicate *gates* merging on + columns already present on the input rows; it does not synthesize a + statistic. Coverage depth, for example, must already be a column on the + rows (typically produced upstream by :ref:`DISJOIN ` and + aggregation). + - **Pairwise only, with single-linkage drift.** The predicate compares each + interval to its immediate sorted predecessor (everything ``LAG`` can + express). Whole-cluster conditions are out of scope. When the predicate is + not an equivalence relation (e.g. ``ABS(score - prev.score) < 5``), + consecutive pairs may each satisfy it while the cluster's extremes do not + — the same single-linkage behavior that ``distance``-based clustering + already exhibits. + Return Value ~~~~~~~~~~~~ @@ -101,6 +134,19 @@ Cluster intervals separately by strand: FROM features ORDER BY chrom, strand, start +**Predicate-Gated Clustering:** + +Cut adjacent intervals into clusters wherever a column's value changes +(run-length encoding). The ``prev.*`` qualifier references the predecessor row: + +.. code-block:: sql + + SELECT + *, + CLUSTER(interval, predicate := depth = prev.depth) AS cluster_id + FROM features + ORDER BY chrom, start + **Analyze Cluster Statistics:** Count features per cluster: @@ -194,6 +240,9 @@ Syntax -- Strand-specific merge SELECT MERGE(interval, stranded := true) FROM features + -- Predicate-gated merge (merge only equal-valued adjacent runs) + SELECT MERGE(interval, predicate := depth = prev.depth) FROM features + -- Merge with additional aggregations SELECT MERGE(interval), @@ -214,6 +263,16 @@ Parameters **stranded** *(optional)* When ``true``, merge intervals separately by strand. Default: ``false``. +**predicate** *(optional)* + A boolean expression that further restricts which adjacent intervals are + merged. ``MERGE`` decomposes into :ref:`CLUSTER ` plus a + ``GROUP BY`` over the cluster id, so it inherits predicate-aware boundaries + directly — see the :ref:`CLUSTER predicate ` description + for the full semantics, the ``prev.*`` qualifier convention, the + references-existing-columns-only constraint, and the pairwise-only / + single-linkage caveat. Omitting the predicate preserves the default + adjacency-only merge. + Return Value ~~~~~~~~~~~~ @@ -256,6 +315,24 @@ Merge intervals separately by strand: SELECT MERGE(interval, stranded := true) FROM features +**Predicate-Gated Merge (coverage depth):** + +Merge only adjacent intervals that share the same coverage depth, reconstructing +a re-clustered, depth-segmented partition from per-breakpoint segments produced +by :ref:`DISJOIN ` and aggregation: + +.. code-block:: sql + + SELECT MERGE(interval, predicate := depth = prev.depth) + FROM ( + SELECT disjoin_chrom AS chrom, + disjoin_start AS start, + disjoin_end AS end, + COUNT(*) AS depth + FROM DISJOIN(features) + GROUP BY disjoin_chrom, disjoin_start, disjoin_end + ) AS segments + **Merge with Feature Count:** Count how many features were merged into each region: diff --git a/docs/recipes/clustering.rst b/docs/recipes/clustering.rst index 8703021..a8dfcd2 100644 --- a/docs/recipes/clustering.rst +++ b/docs/recipes/clustering.rst @@ -347,6 +347,86 @@ Compare raw vs merged coverage: **Use case:** Quantify the redundancy in your feature set. +Predicate-Gated Clustering and Merging +-------------------------------------- + +Both ``CLUSTER`` and ``MERGE`` accept an optional ``predicate :=`` argument that +further restricts which adjacent intervals are coalesced: an interval stays in +the current cluster only when it is adjacent to its predecessor *and* the +predicate holds between the two. Bare columns resolve to the current interval; +the predecessor's value is referenced with a ``prev.*`` qualifier. The predicate +references columns already present on the rows — it gates merging, it does not +synthesize a statistic — and it compares each interval only to its immediate +sorted predecessor (so non-equivalence predicates exhibit single-linkage drift, +just like ``distance``-based clustering). + +Run-Length Encoding on a Column +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Merge only adjacent intervals that share the same value, cutting a new region +wherever the value changes: + +.. code-block:: sql + + SELECT MERGE(interval, predicate := depth = prev.depth) + FROM segments + +**Use case:** Collapse a per-base or per-segment signal into maximal runs of +constant value (e.g. equal coverage depth, same genotype, same annotation). + +Run-Length Encode with CLUSTER +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Assign a distinct cluster id to each maximal equal-valued run while keeping the +individual rows: + +.. code-block:: sql + + SELECT + *, + CLUSTER(interval, predicate := depth = prev.depth) AS run_id + FROM segments + ORDER BY chrom, start + +**Use case:** Label run boundaries for inspection before aggregating. + +Reconstruct disjoin() Coverage Segments +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +GIQL's :ref:`DISJOIN ` primitive splits overlapping intervals +at every breakpoint but deliberately does not re-cluster the resulting +sub-intervals. Pairing it with a predicate-gated ``MERGE`` closes that gap: cut +the input at every breakpoint, aggregate coverage depth per segment, then merge +back the contiguous runs of equal depth — reproducing the re-clustered, +depth-annotated output Bioconductor's ``disjoin()`` produces: + +.. code-block:: sql + + SELECT MERGE(interval, predicate := depth = prev.depth) + FROM ( + SELECT disjoin_chrom AS chrom, + disjoin_start AS start, + disjoin_end AS end, + COUNT(*) AS depth + FROM DISJOIN(features) + GROUP BY disjoin_chrom, disjoin_start, disjoin_end + ) AS segments + +**Use case:** Build a re-clustered coverage profile from overlapping intervals, +the expression-based generalization of ``disjoin()`` to any pairwise condition. + +Multi-Column Predicate +~~~~~~~~~~~~~~~~~~~~~~~ + +Gate merging on more than one column by combining comparisons with ``AND``: + +.. code-block:: sql + + SELECT MERGE(interval, predicate := strand = prev.strand AND name = prev.name) + FROM features + +**Use case:** Keep merged regions homogeneous across several attributes at once. + Advanced Patterns ----------------- diff --git a/src/giql/mcp/server.py b/src/giql/mcp/server.py index 194abb7..3fb2590 100644 --- a/src/giql/mcp/server.py +++ b/src/giql/mcp/server.py @@ -105,6 +105,14 @@ "description": "Max gap to consider same cluster (default: 0)", }, {"name": "stranded", "description": "Cluster by strand (default: false)"}, + { + "name": "predicate", + "description": ( + "Pairwise boolean gate; keep adjacent intervals together " + "only when it holds. Use prev.col for the predecessor row " + "(e.g. predicate := depth = prev.depth). Optional." + ), + }, ], "returns": "Integer cluster ID", "example": "SELECT *, CLUSTER(interval) AS cluster_id FROM features", @@ -118,6 +126,14 @@ {"name": "interval", "description": "Genomic column to merge"}, {"name": "distance", "description": "Max gap to merge (default: 0)"}, {"name": "stranded", "description": "Merge by strand (default: false)"}, + { + "name": "predicate", + "description": ( + "Pairwise boolean gate; merge adjacent intervals only when " + "it holds. Use prev.col for the predecessor row (e.g. " + "predicate := depth = prev.depth). Optional." + ), + }, ], "returns": "Merged interval coordinates (chromosome, start_pos, end_pos)", "example": "SELECT MERGE(interval), COUNT(*) FROM features", From f5766c1738d67a8f9a5fb1f3c540c0ac4d1f93f3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 13:15:26 -0400 Subject: [PATCH 058/142] fix: Reference the predecessor with PREV() and quote predicate columns Address review findings on the CLUSTER/MERGE predicate. The predecessor reference moves from a prev.col qualifier to a PREV(col) function call: a function call cannot collide with a table aliased prev, eliminating the silent-hijack ambiguity the qualifier had. PREV() arguments and bare current-row columns in the predicate are now quoted, so reserved-word genomic columns such as start and end are emitted as valid SQL instead of raising an engine parser error. PREV() calls are validated to take exactly one argument and reject nesting, enforcing the pairwise-only contract. Columns referenced by the predicate are folded into the window's required columns so they are projected into the lag_calc CTE, making the scope dependency explicit and keeping the columns available when a later operator wraps the query in a further subquery. --- src/giql/transformer.py | 60 ++++++++++++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/src/giql/transformer.py b/src/giql/transformer.py index 66d64b2..c757aaf 100644 --- a/src/giql/transformer.py +++ b/src/giql/transformer.py @@ -452,11 +452,11 @@ def _transform_for_cluster( # An optional predicate further restricts which adjacent intervals # are kept together: a row stays in the current cluster only when it # is adjacent to its predecessor AND the predicate holds between them. - # ``prev.col`` references in the predicate resolve to the predecessor + # ``PREV(col)`` references in the predicate resolve to the predecessor # row via LAG over the same partition/order as the adjacency window. predicate_expr = cluster_expr.args.get("predicate") if predicate_expr is not None: - rewritten_predicate = self._rewrite_prev_refs( + rewritten_predicate = self._rewrite_predecessor_refs( predicate_expr, partition_cols, order_by ) keep_together = exp.And( @@ -495,6 +495,14 @@ def _transform_for_cluster( if stranded: required_cols.add(strand_col) + # The predicate is evaluated inside the lag_calc CTE, so every column + # it references (current-row columns and PREV() arguments alike) must + # be projected into that CTE. Folding them into required_cols makes the + # scope dependency explicit and keeps the columns available even when a + # later operator wraps this query in a further subquery. + if predicate_expr is not None: + required_cols |= {col.name for col in predicate_expr.find_all(exp.Column)} + # Check if required columns are already in the select list selected_cols = set() for expr in cte_expressions: @@ -570,19 +578,23 @@ def _transform_for_cluster( return new_query - def _rewrite_prev_refs( + def _rewrite_predecessor_refs( self, predicate: exp.Expression, partition_cols: list[exp.Expression], order_by: list[exp.Ordered], ) -> exp.Expression: - """Rewrite ``prev.col`` references in a predicate to LAG windows. + """Rewrite ``PREV(col)`` calls in a predicate to LAG windows. - Bare column references in the predicate denote the current interval and - are left untouched. Each ``prev.col`` reference denotes the sorted - predecessor and is rewritten to ``LAG(col) OVER (...)`` using the same + Bare column references in the predicate denote the current interval. + Each ``PREV(col)`` call denotes the sorted predecessor's value of that + column and is rewritten to ``LAG(col) OVER (...)`` using the same partition/order as the cluster's adjacency window, so the predicate is - evaluated pairwise against the immediately preceding row. + evaluated pairwise against the immediately preceding row. Every column + identifier (current-row columns and LAG arguments alike) is quoted so + that reserved-word genomic columns such as ``start`` / ``end`` are + emitted as valid SQL, matching how the rest of this transformer quotes + genomic columns. :param predicate: Boolean predicate expression to rewrite (not mutated). @@ -591,22 +603,40 @@ def _rewrite_prev_refs( :param order_by: Window ORDER BY terms (start position). :return: - A copy of the predicate with every ``prev.*`` reference replaced by - an equivalent LAG window. + A copy of the predicate with every ``PREV(...)`` call replaced by an + equivalent LAG window and all column identifiers quoted. + :raises ValueError: + If a ``PREV()`` call does not take exactly one argument, or if a + ``PREV()`` call is nested inside another (predicates compare only + the immediate predecessor). """ + def _is_prev(node: exp.Expression) -> bool: + return isinstance(node, exp.Anonymous) and node.name.upper() == "PREV" + def _replace(node: exp.Expression) -> exp.Expression: - if isinstance(node, exp.Column) and node.table == "prev": + if _is_prev(node): + args = node.expressions + if len(args) != 1: + raise ValueError( + f"PREV() takes exactly one column argument; got {len(args)}." + ) + if any(_is_prev(inner) for inner in args[0].find_all(exp.Anonymous)): + raise ValueError( + "PREV() cannot be nested; a CLUSTER/MERGE predicate " + "compares only the immediate predecessor." + ) return exp.Window( - this=exp.Anonymous( - this="LAG", expressions=[exp.Column(this=node.this.copy())] - ), + this=exp.Anonymous(this="LAG", expressions=[args[0].copy()]), partition_by=[col.copy() for col in partition_cols], order=exp.Order(expressions=[term.copy() for term in order_by]), ) return node - return predicate.copy().transform(_replace) + rewritten = predicate.copy().transform(_replace) + for column in rewritten.find_all(exp.Column): + column.this.set("quoted", True) + return rewritten class MergeTransformer: From dc9dd78c4211e6ba4a1cd2686c5290a07c247c32 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 13:15:33 -0400 Subject: [PATCH 059/142] docs: Update predecessor-reference docs to PREV() Reflect the PREV(col) predecessor syntax in the operator docstrings, the aggregation-operator reference, the clustering recipe, and the MCP operator catalog. Replace every prev.col qualifier reference with PREV(col) and update the prose describing how the predecessor's value is referenced. --- docs/dialect/aggregation-operators.rst | 18 +++++++++--------- docs/recipes/clustering.rst | 10 +++++----- src/giql/expressions.py | 18 +++++++++--------- src/giql/mcp/server.py | 8 ++++---- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/docs/dialect/aggregation-operators.rst b/docs/dialect/aggregation-operators.rst index 5c9893d..f34d1c0 100644 --- a/docs/dialect/aggregation-operators.rst +++ b/docs/dialect/aggregation-operators.rst @@ -41,7 +41,7 @@ Syntax CLUSTER(interval, stranded := true) AS cluster_id -- Predicate-gated clustering (run-length encoding on a column) - CLUSTER(interval, predicate := depth = prev.depth) AS cluster_id + CLUSTER(interval, predicate := depth = PREV(depth)) AS cluster_id -- Combined parameters CLUSTER(interval, distance, stranded := true) AS cluster_id @@ -69,8 +69,8 @@ Parameters Omitting the predicate preserves the default adjacency-only behavior. Bare column references resolve to the *current* interval; the predecessor's - value of a column is referenced with a ``prev.*`` qualifier - (e.g. ``depth = prev.depth``). The predicate composes with ``distance`` and + value of a column is referenced with ``PREV(column)`` + (e.g. ``depth = PREV(depth)``). The predicate composes with ``distance`` and ``stranded`` and is evaluated under the operator's existing per-chromosome (and per-strand) partition and start-position order. @@ -84,7 +84,7 @@ Parameters - **Pairwise only, with single-linkage drift.** The predicate compares each interval to its immediate sorted predecessor (everything ``LAG`` can express). Whole-cluster conditions are out of scope. When the predicate is - not an equivalence relation (e.g. ``ABS(score - prev.score) < 5``), + not an equivalence relation (e.g. ``ABS(score - PREV(score)) < 5``), consecutive pairs may each satisfy it while the cluster's extremes do not — the same single-linkage behavior that ``distance``-based clustering already exhibits. @@ -137,13 +137,13 @@ Cluster intervals separately by strand: **Predicate-Gated Clustering:** Cut adjacent intervals into clusters wherever a column's value changes -(run-length encoding). The ``prev.*`` qualifier references the predecessor row: +(run-length encoding). ``PREV(column)`` references the predecessor row's value: .. code-block:: sql SELECT *, - CLUSTER(interval, predicate := depth = prev.depth) AS cluster_id + CLUSTER(interval, predicate := depth = PREV(depth)) AS cluster_id FROM features ORDER BY chrom, start @@ -241,7 +241,7 @@ Syntax SELECT MERGE(interval, stranded := true) FROM features -- Predicate-gated merge (merge only equal-valued adjacent runs) - SELECT MERGE(interval, predicate := depth = prev.depth) FROM features + SELECT MERGE(interval, predicate := depth = PREV(depth)) FROM features -- Merge with additional aggregations SELECT @@ -268,7 +268,7 @@ Parameters merged. ``MERGE`` decomposes into :ref:`CLUSTER ` plus a ``GROUP BY`` over the cluster id, so it inherits predicate-aware boundaries directly — see the :ref:`CLUSTER predicate ` description - for the full semantics, the ``prev.*`` qualifier convention, the + for the full semantics, the ``PREV(column)`` convention, the references-existing-columns-only constraint, and the pairwise-only / single-linkage caveat. Omitting the predicate preserves the default adjacency-only merge. @@ -323,7 +323,7 @@ by :ref:`DISJOIN ` and aggregation: .. code-block:: sql - SELECT MERGE(interval, predicate := depth = prev.depth) + SELECT MERGE(interval, predicate := depth = PREV(depth)) FROM ( SELECT disjoin_chrom AS chrom, disjoin_start AS start, diff --git a/docs/recipes/clustering.rst b/docs/recipes/clustering.rst index a8dfcd2..f56c85e 100644 --- a/docs/recipes/clustering.rst +++ b/docs/recipes/clustering.rst @@ -354,7 +354,7 @@ Both ``CLUSTER`` and ``MERGE`` accept an optional ``predicate :=`` argument that further restricts which adjacent intervals are coalesced: an interval stays in the current cluster only when it is adjacent to its predecessor *and* the predicate holds between the two. Bare columns resolve to the current interval; -the predecessor's value is referenced with a ``prev.*`` qualifier. The predicate +the predecessor's value is referenced with ``PREV(column)``. The predicate references columns already present on the rows — it gates merging, it does not synthesize a statistic — and it compares each interval only to its immediate sorted predecessor (so non-equivalence predicates exhibit single-linkage drift, @@ -368,7 +368,7 @@ wherever the value changes: .. code-block:: sql - SELECT MERGE(interval, predicate := depth = prev.depth) + SELECT MERGE(interval, predicate := depth = PREV(depth)) FROM segments **Use case:** Collapse a per-base or per-segment signal into maximal runs of @@ -384,7 +384,7 @@ individual rows: SELECT *, - CLUSTER(interval, predicate := depth = prev.depth) AS run_id + CLUSTER(interval, predicate := depth = PREV(depth)) AS run_id FROM segments ORDER BY chrom, start @@ -402,7 +402,7 @@ depth-annotated output Bioconductor's ``disjoin()`` produces: .. code-block:: sql - SELECT MERGE(interval, predicate := depth = prev.depth) + SELECT MERGE(interval, predicate := depth = PREV(depth)) FROM ( SELECT disjoin_chrom AS chrom, disjoin_start AS start, @@ -422,7 +422,7 @@ Gate merging on more than one column by combining comparisons with ``AND``: .. code-block:: sql - SELECT MERGE(interval, predicate := strand = prev.strand AND name = prev.name) + SELECT MERGE(interval, predicate := strand = PREV(strand) AND name = PREV(name)) FROM features **Use case:** Keep merged regions homogeneous across several attributes at once. diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 5143d31..c6894d1 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -203,21 +203,21 @@ class GIQLCluster(exp.Func): between each interval and its sorted predecessor; intervals are only kept in the same cluster when they are adjacent *and* the predicate holds. Bare columns resolve to the current interval; the predecessor's value of a - column is referenced with a ``prev.*`` qualifier. + column is referenced with ``PREV(column)``. Examples: CLUSTER(interval) CLUSTER(interval, 1000) CLUSTER(interval, stranded := true) CLUSTER(interval, 1000, stranded := true) - CLUSTER(interval, predicate := depth = prev.depth) + CLUSTER(interval, predicate := depth = PREV(depth)) """ arg_types = { "this": True, # genomic column "distance": False, # maximum distance between features "stranded": False, # strand-specific clustering - "predicate": False, # pairwise boolean gate (current row vs prev.*) + "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } @classmethod @@ -242,23 +242,23 @@ class GIQLMerge(exp.Func): The optional ``predicate`` argument gates merging on a pairwise boolean expression between each interval and its sorted predecessor (see - :class:`GIQLCluster`); a ``prev.*`` qualifier references the predecessor's - value of a column. When the predicate tests equality of a value this - yields a run-length encoding of the input interval sequence. + :class:`GIQLCluster`); ``PREV(column)`` references the predecessor's value + of a column. When the predicate tests equality of a value this yields a + run-length encoding of the input interval sequence. Examples: MERGE(interval) MERGE(interval, 1000) MERGE(interval, stranded := true) - MERGE(interval, predicate := depth = prev.depth) - MERGE(interval, predicate := strand = prev.strand AND name = prev.name) + MERGE(interval, predicate := depth = PREV(depth)) + MERGE(interval, predicate := strand = PREV(strand) AND name = PREV(name)) """ arg_types = { "this": True, # genomic column "distance": False, # maximum distance between features "stranded": False, # strand-specific merging - "predicate": False, # pairwise boolean gate (current row vs prev.*) + "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } @classmethod diff --git a/src/giql/mcp/server.py b/src/giql/mcp/server.py index 3fb2590..08fb7a8 100644 --- a/src/giql/mcp/server.py +++ b/src/giql/mcp/server.py @@ -109,8 +109,8 @@ "name": "predicate", "description": ( "Pairwise boolean gate; keep adjacent intervals together " - "only when it holds. Use prev.col for the predecessor row " - "(e.g. predicate := depth = prev.depth). Optional." + "only when it holds. Use PREV(col) for the predecessor row's " + "value (e.g. predicate := depth = PREV(depth)). Optional." ), }, ], @@ -130,8 +130,8 @@ "name": "predicate", "description": ( "Pairwise boolean gate; merge adjacent intervals only when " - "it holds. Use prev.col for the predecessor row (e.g. " - "predicate := depth = prev.depth). Optional." + "it holds. Use PREV(col) for the predecessor row's value " + "(e.g. predicate := depth = PREV(depth)). Optional." ), }, ], From 88304f064e20ace3042a7c2c12216b1f8853130f Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 13:15:39 -0400 Subject: [PATCH 060/142] test: Cover PREV() predecessor syntax and review-finding fixes Update the parsing, transpilation, and integration tests to the PREV(col) syntax and assert the quoted LAG rewrite. Add coverage for the reserved-word predicate columns (transpile and DuckDB execution), the PREV() arity and nesting errors, and a predicate column absent from the projection. Make the stranded-MERGE assertion order-independent, since the emitted SQL orders only by chrom and start and the two same-start rows have no defined order. Drop the dead module-level marker from the integration conftest. --- .../integration/cluster_predicate/conftest.py | 4 +- .../test_cluster_predicate.py | 105 ++++++++++++++---- tests/test_cluster_parsing.py | 30 ++--- tests/test_cluster_predicate_transpilation.py | 92 ++++++++++++--- tests/test_merge_parsing.py | 22 ++-- 5 files changed, 186 insertions(+), 67 deletions(-) diff --git a/tests/integration/cluster_predicate/conftest.py b/tests/integration/cluster_predicate/conftest.py index c6a00e3..6b04518 100644 --- a/tests/integration/cluster_predicate/conftest.py +++ b/tests/integration/cluster_predicate/conftest.py @@ -12,8 +12,6 @@ duckdb = pytest.importorskip("duckdb") -pytestmark = pytest.mark.integration - from tests.integration.bedtools.utils.duckdb_loader import load_intervals # noqa: E402 @@ -32,7 +30,7 @@ def giql_query(duckdb_connection): Usage:: rows = giql_query( - "SELECT MERGE(interval, predicate := score = prev.score) FROM t", + "SELECT MERGE(interval, predicate := score = PREV(score)) FROM t", tables=["t"], t=[("chr1", 0, 10, "a", 5, "+"), ...], ) diff --git a/tests/integration/cluster_predicate/test_cluster_predicate.py b/tests/integration/cluster_predicate/test_cluster_predicate.py index dead147..8cbc528 100644 --- a/tests/integration/cluster_predicate/test_cluster_predicate.py +++ b/tests/integration/cluster_predicate/test_cluster_predicate.py @@ -52,13 +52,13 @@ def test_merge_with_equality_predicate_run_length_encodes(self, giql_query): Given: Five abutting intervals whose scores form the runs 5,5 | 3,3 | 5. When: - MERGE(interval, predicate := score = prev.score) runs. + MERGE(interval, predicate := score = PREV(score)) runs. Then: It should emit one merged interval per maximal equal-score run. """ # Arrange & act rows = giql_query( - "SELECT MERGE(interval, predicate := score = prev.score) FROM intervals", + "SELECT MERGE(interval, predicate := score = PREV(score)) FROM intervals", tables=["intervals"], intervals=[ _ival("chr1", 0, 10, "a", 5), @@ -83,14 +83,14 @@ def test_merge_predicate_drifts_under_single_linkage(self, giql_query): Three abutting intervals with scores 10, 13, 16, where each consecutive pair differs by 3 but the extremes differ by 6. When: - MERGE(interval, predicate := ABS(score - prev.score) < 5) runs. + MERGE(interval, predicate := ABS(score - PREV(score)) < 5) runs. Then: It should merge the entire run into one interval even though the cluster's extremes violate the predicate (documented drift). """ # Arrange & act rows = giql_query( - "SELECT MERGE(interval, predicate := ABS(score - prev.score) < 5) " + "SELECT MERGE(interval, predicate := ABS(score - PREV(score)) < 5) " "FROM intervals", tables=["intervals"], intervals=[ @@ -110,16 +110,16 @@ def test_merge_compound_predicate_breaks_on_either_column(self, giql_query): Four abutting intervals where the (name, strand) pair changes first on strand and then on name partway through the run. When: - MERGE(interval, predicate := strand = prev.strand AND - name = prev.name) runs. + MERGE(interval, predicate := strand = PREV(strand) AND + name = PREV(name)) runs. Then: It should break the merge wherever either column changes, yielding one merged interval per homogeneous (name, strand) run. """ # Arrange & act rows = giql_query( - "SELECT MERGE(interval, predicate := strand = prev.strand " - "AND name = prev.name) FROM intervals", + "SELECT MERGE(interval, predicate := strand = PREV(strand) " + "AND name = PREV(name)) FROM intervals", tables=["intervals"], intervals=[ _ival("chr1", 0, 10, "g", 5, "+"), @@ -142,7 +142,7 @@ def test_merge_stranded_predicate_evaluates_within_strand(self, giql_query): Given: Equal-name abutting intervals on both the + and - strands. When: - MERGE(interval, stranded := true, predicate := name = prev.name) + MERGE(interval, stranded := true, predicate := name = PREV(name)) runs. Then: It should merge each strand's equal-name run independently, emitting @@ -150,7 +150,7 @@ def test_merge_stranded_predicate_evaluates_within_strand(self, giql_query): """ # Arrange & act rows = giql_query( - "SELECT MERGE(interval, stranded := true, predicate := name = prev.name) " + "SELECT MERGE(interval, stranded := true, predicate := name = PREV(name)) " "FROM intervals", tables=["intervals"], intervals=[ @@ -162,10 +162,15 @@ def test_merge_stranded_predicate_evaluates_within_strand(self, giql_query): ) # Assert - assert rows == [ - ("chr1", "+", 0, 20), - ("chr1", "-", 0, 20), - ] + # The emitted MERGE SQL orders only by (chrom, start); these two rows + # tie on (chr1, 0) and differ only by strand, so the row order is not + # deterministic. Compare order-independently. + assert sorted(rows) == sorted( + [ + ("chr1", "+", 0, 20), + ("chr1", "-", 0, 20), + ] + ) class TestClusterPredicate: @@ -177,14 +182,14 @@ def test_cluster_with_equality_predicate_assigns_run_ids(self, giql_query): Given: Five abutting intervals whose scores form the runs 5,5 | 3,3 | 5. When: - CLUSTER(interval, predicate := score = prev.score) runs. + CLUSTER(interval, predicate := score = PREV(score)) runs. Then: It should assign a distinct cluster id to each maximal equal-score run, comparing each row only to its immediate predecessor. """ # Arrange & act rows = giql_query( - "SELECT name, CLUSTER(interval, predicate := score = prev.score) AS cid " + "SELECT name, CLUSTER(interval, predicate := score = PREV(score)) AS cid " "FROM intervals", tables=["intervals"], intervals=[ @@ -208,7 +213,7 @@ def test_cluster_distance_and_predicate_compose(self, giql_query): Given: Three equal-score intervals separated by a 50bp gap then a 150bp gap, clustered with CLUSTER(interval, 100, predicate := score = - prev.score). + PREV(score)). When: The generated SQL runs in DuckDB. Then: @@ -218,7 +223,7 @@ def test_cluster_distance_and_predicate_compose(self, giql_query): """ # Arrange & act rows = giql_query( - "SELECT name, CLUSTER(interval, 100, predicate := score = prev.score) " + "SELECT name, CLUSTER(interval, 100, predicate := score = PREV(score)) " "AS cid FROM intervals", tables=["intervals"], intervals=[ @@ -238,7 +243,7 @@ def test_cluster_predicate_resets_at_chromosome_boundary(self, giql_query): Given: Equal-score abutting intervals on chr1 and chr2, clustered with - CLUSTER(interval, predicate := score = prev.score). + CLUSTER(interval, predicate := score = PREV(score)). When: The generated SQL runs in DuckDB. Then: @@ -248,7 +253,7 @@ def test_cluster_predicate_resets_at_chromosome_boundary(self, giql_query): """ # Arrange & act rows = giql_query( - "SELECT chrom, name, CLUSTER(interval, predicate := score = prev.score) " + "SELECT chrom, name, CLUSTER(interval, predicate := score = PREV(score)) " "AS cid FROM intervals", tables=["intervals"], intervals=[ @@ -267,6 +272,62 @@ def test_cluster_predicate_resets_at_chromosome_boundary(self, giql_query): # chr2's first row starts its own cluster (ids are per-partition). assert {by_name["a"], by_name["c"]} == {("chr1", 1), ("chr2", 1)} + def test_cluster_reserved_word_predicate_executes(self, giql_query): + """Test that a predicate over reserved-word columns executes. + + Given: + Three abutting intervals and a predicate over the reserved-word + genomic columns start and end (start = PREV(end)). + When: + The generated SQL runs in DuckDB. + Then: + It should emit valid quoted SQL and merge the abutting run, proving + reserved-word predicate columns are handled. + """ + # Arrange & act + rows = giql_query( + "SELECT MERGE(interval, predicate := start = PREV(end)) FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 5), + _ival("chr1", 10, 20, "b", 5), + _ival("chr1", 20, 30, "c", 5), + ], + ) + + # Assert + assert rows == [("chr1", 0, 30)] + + def test_cluster_predicate_column_absent_from_projection_executes(self, giql_query): + """Test that a predicate column need not appear in the projection. + + Given: + A CLUSTER query whose explicit projection omits the predicate + column score (selecting only chrom, start, end, and the cluster id). + When: + The generated SQL runs in DuckDB. + Then: + It should still cluster by the score runs, confirming the predicate + column is projected into the intermediate CTE. + """ + # Arrange & act + rows = giql_query( + 'SELECT chrom, start, "end", ' + "CLUSTER(interval, predicate := score = PREV(score)) AS cid " + "FROM intervals", + tables=["intervals"], + intervals=[ + _ival("chr1", 0, 10, "a", 5), + _ival("chr1", 10, 20, "b", 5), + _ival("chr1", 20, 30, "c", 3), + ], + ) + + # Assert + cids = [row[-1] for row in rows] + assert cids[0] == cids[1] + assert cids[2] != cids[0] + class TestDisjoinDepthMergePipeline: """DISJOIN -> depth-aggregation -> predicate-MERGE reconstructs disjoin().""" @@ -321,7 +382,7 @@ def test_predicate_merge_remerges_equal_depth_segments(self, giql_query): Given: The depth-annotated disjoint segments (depths 2, 3, 3, 2). When: - MERGE(interval, predicate := depth = prev.depth) runs over them. + MERGE(interval, predicate := depth = PREV(depth)) runs over them. Then: It should coalesce the adjacent depth-3 run into [10, 30) while keeping the depth-2 flanks distinct, reproducing the re-clustered @@ -329,7 +390,7 @@ def test_predicate_merge_remerges_equal_depth_segments(self, giql_query): """ # Arrange & act rows = giql_query( - "SELECT MERGE(interval, predicate := depth = prev.depth) " + "SELECT MERGE(interval, predicate := depth = PREV(depth)) " f"FROM ({self._DEPTH_SEGMENTS_QUERY}) AS segments", tables=["features"], features=self._FEATURES, diff --git a/tests/test_cluster_parsing.py b/tests/test_cluster_parsing.py index 0dd7ec6..b66cd12 100644 --- a/tests/test_cluster_parsing.py +++ b/tests/test_cluster_parsing.py @@ -101,7 +101,7 @@ def test_from_arg_list_with_predicate(self): """Test that a predicate named argument is captured on the node. Given: - A GIQL query with CLUSTER(interval, predicate := depth = prev.depth). + A GIQL query with CLUSTER(interval, predicate := depth = PREV(depth)). When: Parsing the query. Then: @@ -109,7 +109,7 @@ def test_from_arg_list_with_predicate(self): """ # Act ast = parse_one( - "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "SELECT *, CLUSTER(interval, predicate := depth = PREV(depth)) AS cid " "FROM peaks", dialect=GIQLDialect, ) @@ -119,36 +119,36 @@ def test_from_arg_list_with_predicate(self): assert isinstance(cluster_expr, GIQLCluster) assert isinstance(cluster_expr.args.get("predicate"), exp.EQ) - def test_from_arg_list_with_predicate_prev_qualifier(self): - """Test that a prev. qualifier parses as a column on the predecessor. + def test_from_arg_list_with_predicate_prev_call(self): + """Test that a PREV() call parses as a predecessor function reference. Given: - A predicate CLUSTER(interval, predicate := depth = prev.depth) - referencing the predecessor row with the prev. qualifier. + A predicate CLUSTER(interval, predicate := depth = PREV(depth)) + referencing the predecessor row with the PREV() function. When: Parsing the query. Then: - It should parse prev.depth as a column whose table is prev. + It should parse PREV(depth) as an anonymous PREV call over depth. """ # Act ast = parse_one( - "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "SELECT *, CLUSTER(interval, predicate := depth = PREV(depth)) AS cid " "FROM peaks", dialect=GIQLDialect, ) # Assert predicate = ast.expressions[1].this.args["predicate"] - prev_ref = predicate.expression - assert isinstance(prev_ref, exp.Column) - assert prev_ref.table == "prev" - assert prev_ref.name == "depth" + prev_call = predicate.expression + assert isinstance(prev_call, exp.Anonymous) + assert prev_call.name.upper() == "PREV" + assert [arg.name for arg in prev_call.expressions] == ["depth"] def test_from_arg_list_with_predicate_kwarg_syntax(self): """Test that the => kwarg form also binds the predicate argument. Given: - A CLUSTER call using CLUSTER(interval, predicate => score = prev.score) + A CLUSTER call using CLUSTER(interval, predicate => score = PREV(score)) with the => kwarg form rather than :=. When: Parsing the query. @@ -157,7 +157,7 @@ def test_from_arg_list_with_predicate_kwarg_syntax(self): """ # Act ast = parse_one( - "SELECT *, CLUSTER(interval, predicate => score = prev.score) AS cid " + "SELECT *, CLUSTER(interval, predicate => score = PREV(score)) AS cid " "FROM peaks", dialect=GIQLDialect, ) @@ -181,7 +181,7 @@ def test_from_arg_list_with_predicate_and_positional_distance(self): # Act ast = parse_one( "SELECT *, CLUSTER(interval, 1000, stranded := true, " - "predicate := name = prev.name) AS cid FROM peaks", + "predicate := name = PREV(name)) AS cid FROM peaks", dialect=GIQLDialect, ) diff --git a/tests/test_cluster_predicate_transpilation.py b/tests/test_cluster_predicate_transpilation.py index f553543..93be367 100644 --- a/tests/test_cluster_predicate_transpilation.py +++ b/tests/test_cluster_predicate_transpilation.py @@ -1,11 +1,13 @@ """Transpilation tests for the CLUSTER/MERGE predicate argument. Tests verify that an optional ``predicate :=`` argument ANDs into the -cluster-boundary CASE, that ``prev.col`` references are rewritten to LAG +cluster-boundary CASE, that ``PREV(col)`` calls are rewritten to quoted LAG windows over the operator's partition/order, and that omitting the predicate leaves the emitted SQL byte-identical to the pre-predicate behavior. """ +import pytest + from giql import transpile @@ -37,7 +39,7 @@ def test_transpile_with_predicate_ands_into_case(self): """Test that a predicate ANDs into the cluster-boundary CASE. Given: - A CLUSTER query with predicate := depth = prev.depth. + A CLUSTER query with predicate := depth = PREV(depth). When: Transpiling to SQL. Then: @@ -45,7 +47,7 @@ def test_transpile_with_predicate_ands_into_case(self): """ # Act sql = transpile( - "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "SELECT *, CLUSTER(interval, predicate := depth = PREV(depth)) AS cid " "FROM peaks", tables=["peaks"], ) @@ -53,35 +55,35 @@ def test_transpile_with_predicate_ands_into_case(self): # Assert assert ' >= "start" AND (' in sql - def test_transpile_rewrites_prev_ref_to_lag_window(self): - """Test that a prev. reference becomes a LAG over the cluster window. + def test_transpile_rewrites_prev_call_to_lag_window(self): + """Test that a PREV() call becomes a quoted LAG over the cluster window. Given: - A CLUSTER query with predicate := depth = prev.depth. + A CLUSTER query with predicate := depth = PREV(depth). When: Transpiling to SQL. Then: - It should rewrite prev.depth to LAG(depth) over the same + It should rewrite PREV(depth) to a quoted LAG("depth") over the same partition/order as the adjacency window. """ # Act sql = transpile( - "SELECT *, CLUSTER(interval, predicate := depth = prev.depth) AS cid " + "SELECT *, CLUSTER(interval, predicate := depth = PREV(depth)) AS cid " "FROM peaks", tables=["peaks"], ) # Assert assert ( - 'depth = LAG(depth) OVER (PARTITION BY "chrom" ORDER BY "start" ' + '"depth" = LAG("depth") OVER (PARTITION BY "chrom" ORDER BY "start" ' "NULLS LAST)" in sql ) def test_transpile_predicate_uses_stranded_partition(self): - """Test that prev. LAG windows honor the stranded partition. + """Test that PREV() LAG windows honor the stranded partition. Given: - A stranded CLUSTER query with a predicate referencing prev.name. + A stranded CLUSTER query with a predicate referencing PREV(name). When: Transpiling to SQL. Then: @@ -90,16 +92,74 @@ def test_transpile_predicate_uses_stranded_partition(self): # Act sql = transpile( "SELECT *, CLUSTER(interval, stranded := true, " - "predicate := name = prev.name) AS cid FROM peaks", + "predicate := name = PREV(name)) AS cid FROM peaks", tables=["peaks"], ) # Assert assert ( - 'LAG(name) OVER (PARTITION BY "chrom", "strand" ORDER BY "start" ' + 'LAG("name") OVER (PARTITION BY "chrom", "strand" ORDER BY "start" ' "NULLS LAST)" in sql ) + def test_transpile_quotes_reserved_word_predicate_columns(self): + """Test that reserved-word predicate columns are quoted on both sides. + + Given: + A CLUSTER query whose predicate references the reserved-word + genomic columns end and start (end = PREV(start)). + When: + Transpiling to SQL. + Then: + It should quote both the current-row column and the LAG argument so + the emitted SQL is valid. + """ + # Act + sql = transpile( + "SELECT *, CLUSTER(interval, predicate := end = PREV(start)) AS cid " + "FROM peaks", + tables=["peaks"], + ) + + # Assert + assert '"end" = LAG("start") OVER (' in sql + + def test_transpile_prev_with_wrong_arity_raises(self): + """Test that a PREV() call with the wrong arity is rejected. + + Given: + A CLUSTER predicate calling PREV() with two arguments. + When: + Transpiling to SQL. + Then: + It should raise a ValueError naming the one-argument requirement. + """ + # Arrange, act, & assert + with pytest.raises(ValueError, match="exactly one column argument"): + transpile( + "SELECT *, CLUSTER(interval, predicate := depth = PREV(depth, score)) " + "AS cid FROM peaks", + tables=["peaks"], + ) + + def test_transpile_nested_prev_raises(self): + """Test that a nested PREV() call is rejected. + + Given: + A CLUSTER predicate nesting PREV() inside another PREV(). + When: + Transpiling to SQL. + Then: + It should raise a ValueError naming the no-nesting restriction. + """ + # Arrange, act, & assert + with pytest.raises(ValueError, match="cannot be nested"): + transpile( + "SELECT *, CLUSTER(interval, predicate := depth = PREV(PREV(depth))) " + "AS cid FROM peaks", + tables=["peaks"], + ) + class TestMergePredicateTranspilation: """Tests for MERGE predicate transpilation to SQL.""" @@ -127,7 +187,7 @@ def test_transpile_predicate_inherited_through_cluster(self): """Test that MERGE inherits predicate-aware cluster boundaries. Given: - A MERGE query with predicate := depth = prev.depth. + A MERGE query with predicate := depth = PREV(depth). When: Transpiling to SQL. Then: @@ -136,13 +196,13 @@ def test_transpile_predicate_inherited_through_cluster(self): """ # Act sql = transpile( - "SELECT MERGE(interval, predicate := depth = prev.depth) FROM peaks", + "SELECT MERGE(interval, predicate := depth = PREV(depth)) FROM peaks", tables=["peaks"], ) # Assert assert ' >= "start" AND (' in sql assert ( - 'depth = LAG(depth) OVER (PARTITION BY "chrom" ORDER BY "start" ' + '"depth" = LAG("depth") OVER (PARTITION BY "chrom" ORDER BY "start" ' "NULLS LAST)" in sql ) diff --git a/tests/test_merge_parsing.py b/tests/test_merge_parsing.py index eb94e3c..1385ec1 100644 --- a/tests/test_merge_parsing.py +++ b/tests/test_merge_parsing.py @@ -91,7 +91,7 @@ def test_from_arg_list_with_predicate(self): """Test that a predicate named argument is captured on the node. Given: - A GIQL query with MERGE(interval, predicate := depth = prev.depth). + A GIQL query with MERGE(interval, predicate := depth = PREV(depth)). When: Parsing the query. Then: @@ -99,7 +99,7 @@ def test_from_arg_list_with_predicate(self): """ # Act ast = parse_one( - "SELECT MERGE(interval, predicate := depth = prev.depth) FROM peaks", + "SELECT MERGE(interval, predicate := depth = PREV(depth)) FROM peaks", dialect=GIQLDialect, ) @@ -112,7 +112,7 @@ def test_from_arg_list_with_predicate_kwarg_syntax(self): """Test that the => kwarg form also binds the predicate argument. Given: - A MERGE call using MERGE(interval, predicate => score = prev.score) + A MERGE call using MERGE(interval, predicate => score = PREV(score)) with the => kwarg form rather than :=. When: Parsing the query. @@ -121,7 +121,7 @@ def test_from_arg_list_with_predicate_kwarg_syntax(self): """ # Act ast = parse_one( - "SELECT MERGE(interval, predicate => score = prev.score) FROM peaks", + "SELECT MERGE(interval, predicate => score = PREV(score)) FROM peaks", dialect=GIQLDialect, ) @@ -134,22 +134,22 @@ def test_from_arg_list_with_compound_predicate(self): """Test that a multi-term predicate parses as a conjunction. Given: - A query MERGE(interval, predicate := strand = prev.strand AND - name = prev.name) joining two pairwise comparisons with AND. + A query MERGE(interval, predicate := strand = PREV(strand) AND + name = PREV(name)) joining two pairwise comparisons with AND. When: Parsing the query. Then: - It should attach the predicate as an AND of two prev. comparisons. + It should attach the predicate as an AND of two PREV() comparisons. """ # Act ast = parse_one( - "SELECT MERGE(interval, predicate := strand = prev.strand " - "AND name = prev.name) FROM peaks", + "SELECT MERGE(interval, predicate := strand = PREV(strand) " + "AND name = PREV(name)) FROM peaks", dialect=GIQLDialect, ) # Assert predicate = ast.expressions[0].args["predicate"] assert isinstance(predicate, exp.And) - prev_tables = {col.table for col in predicate.find_all(exp.Column)} - assert "prev" in prev_tables + prev_calls = {call.name.upper() for call in predicate.find_all(exp.Anonymous)} + assert "PREV" in prev_calls From e1842f023fa5927013c6b57a2c19beda06cc0dd1 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 22:46:04 -0400 Subject: [PATCH 061/142] fix: Add bedtools parity offset to DISTANCE non-overlapping gap DISTANCE returned the raw 0-based half-open gap, so book-ended (adjacent) intervals where A.end equals B.start reported 0, the same value as overlapping intervals. bedtools closest -d, the project's correctness oracle, reports 1 for book-ended features and N+1 for a raw gap of N. Add 1 to the absolute gap magnitude in the two non-overlapping branches of all four variants the distance CASE emits (unsigned, signed, stranded, stranded plus signed). The overlap branch stays 0 and the NULL branches are untouched. The offset is applied before any directional sign, so a downstream book-ended pair reports +1 and an upstream one -1 in signed mode. NEAREST shares the same generator and shifts in lockstep; its ordering is preserved since the offset is monotonic, while its reported distance column and max_distance boundary move by one. --- src/giql/generators/base.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 84dfd34..9038369 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -587,6 +587,13 @@ def _generate_distance_case( ) -> str: """Generate SQL CASE expression for distance calculation. + Distances follow bedtools ``closest -d`` semantics: overlapping + intervals report ``0``, book-ended (adjacent) intervals where + ``A.end == B.start`` in half-open coordinates report ``1``, and a raw + half-open gap of N bases reports ``N + 1``. The ``+ 1`` is applied to + the absolute gap magnitude before any directional sign, so a downstream + book-ended pair reports ``+1`` and an upstream one ``-1`` in signed mode. + :param chrom_a: Chromosome column for interval A :param start_a: @@ -619,15 +626,15 @@ def _generate_distance_case( return ( f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " - f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a}) " - f"ELSE -({start_a} - {end_b}) END" + f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " + f"ELSE -({start_a} - {end_b} + 1) END" ) # Unsigned (absolute) distance return ( f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " - f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a}) " - f"ELSE ({start_a} - {end_b}) END" + f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " + f"ELSE ({start_a} - {end_b} + 1) END" ) # Stranded distance calculation @@ -642,11 +649,11 @@ def _generate_distance_case( f"WHEN {strand_b} = '.' OR {strand_b} = '?' THEN NULL " f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " f"WHEN {end_a} <= {start_b} THEN " - f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a}) " - f"ELSE ({start_b} - {end_a}) END " + f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a} + 1) " + f"ELSE ({start_b} - {end_a} + 1) END " f"ELSE " - f"CASE WHEN {strand_a} = '-' THEN ({start_a} - {end_b}) " - f"ELSE -({start_a} - {end_b}) END END" + f"CASE WHEN {strand_a} = '-' THEN ({start_a} - {end_b} + 1) " + f"ELSE -({start_a} - {end_b} + 1) END END" ) # Stranded but not signed: apply strand flip only return ( @@ -656,11 +663,11 @@ def _generate_distance_case( f"WHEN {strand_b} = '.' OR {strand_b} = '?' THEN NULL " f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " f"WHEN {end_a} <= {start_b} THEN " - f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a}) " - f"ELSE ({start_b} - {end_a}) END " + f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a} + 1) " + f"ELSE ({start_b} - {end_a} + 1) END " f"ELSE " - f"CASE WHEN {strand_a} = '-' THEN -({start_a} - {end_b}) " - f"ELSE ({start_a} - {end_b}) END END" + f"CASE WHEN {strand_a} = '-' THEN -({start_a} - {end_b} + 1) " + f"ELSE ({start_a} - {end_b} + 1) END END" ) def _predicate_operand(self, expression: exp.Expression, arg: str) -> ResolvedColumn: From 0b5f3d80ebc51624912fd51deb41b735c45a3e95 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 22:46:19 -0400 Subject: [PATCH 062/142] test: Cover and pin bedtools parity offset for DISTANCE and NEAREST Update the existing distance and nearest assertions to the parity-corrected values (book-ended now 1, a raw gap of N now N+1) and add coverage that fails automatically should the offset regress. New behavioral coverage exercises the magnitude and sign formula end to end: the 1 bp gap, the overlap to book-ended to gap value sweep, the unsigned upstream branch, signed book-ended plus and minus one, the stranded upstream branch, and a stranded plus signed class for the upstream sign divergence and NULL strands. The coordinate-space suite gains mixed-convention invariance for the book-ended and 1 bp boundaries. The bedtools distance suite now cross-validates against live bedtools closest -d across gap sizes instead of hard-coded constants, and the nearest suite asserts the reported distance against the oracle, pins the inclusive max_distance boundary on both sides, and adds Hypothesis properties for distance equals gap plus one, selection invariance under the shift, and max_distance filtering on the shifted distance. A generator test guards against fusing the encoding and parity offsets into one term. --- tests/generators/test_base.py | 147 +++--- tests/integration/bedtools/test_distance.py | 158 +++++-- tests/integration/bedtools/test_nearest.py | 110 +++-- .../bedtools/test_nearest_property.py | 153 ++++++ .../test_distance_coordinate_space.py | 207 +++++++- .../test_nearest_coordinate_space.py | 31 +- tests/test_distance_transpilation.py | 10 +- tests/test_distance_udf.py | 442 +++++++++++++++--- 8 files changed, 1037 insertions(+), 221 deletions(-) create mode 100644 tests/integration/bedtools/test_nearest_property.py diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index 10a2d9c..f95e91b 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -411,16 +411,16 @@ def test_giqlnearest_sql_standalone(self, tables_with_peaks_and_genes): "CASE WHEN 'chr1' != genes.\"chrom\" THEN NULL " 'WHEN 1000 < genes."end" AND 2000 > genes."start" THEN 0 ' 'WHEN 2000 <= genes."start" ' - 'THEN (genes."start" - 2000) ' - 'ELSE (1000 - genes."end") END AS distance\n' + 'THEN (genes."start" - 2000 + 1) ' + 'ELSE (1000 - genes."end" + 1) END AS distance\n' " FROM genes\n" " WHERE 'chr1' = genes.\"chrom\"\n" " ORDER BY ABS(" "CASE WHEN 'chr1' != genes.\"chrom\" THEN NULL " 'WHEN 1000 < genes."end" AND 2000 > genes."start" THEN 0 ' 'WHEN 2000 <= genes."start" ' - 'THEN (genes."start" - 2000) ' - 'ELSE (1000 - genes."end") END)\n' + 'THEN (genes."start" - 2000 + 1) ' + 'ELSE (1000 - genes."end" + 1) END)\n' " LIMIT 3\n" " )" ) @@ -446,8 +446,8 @@ def test_giqlnearest_sql_correlated(self, tables_with_peaks_and_genes): 'WHEN peaks."start" < genes."end" ' 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end") ' - 'ELSE (peaks."start" - genes."end") END AS distance\n' + 'THEN (genes."start" - peaks."end" + 1) ' + 'ELSE (peaks."start" - genes."end" + 1) END AS distance\n' " FROM genes\n" ' WHERE peaks."chrom" = genes."chrom"\n' " ORDER BY ABS(" @@ -455,8 +455,8 @@ def test_giqlnearest_sql_correlated(self, tables_with_peaks_and_genes): 'WHEN peaks."start" < genes."end" ' 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end") ' - 'ELSE (peaks."start" - genes."end") END)\n' + 'THEN (genes."start" - peaks."end" + 1) ' + 'ELSE (peaks."start" - genes."end" + 1) END)\n' " LIMIT 3\n" " )" ) @@ -483,8 +483,8 @@ def test_giqlnearest_sql_with_max_distance(self, tables_with_peaks_and_genes): 'WHEN peaks."start" < genes."end" ' 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end") ' - 'ELSE (peaks."start" - genes."end") END AS distance\n' + 'THEN (genes."start" - peaks."end" + 1) ' + 'ELSE (peaks."start" - genes."end" + 1) END AS distance\n' " FROM genes\n" ' WHERE peaks."chrom" = genes."chrom" ' "AND (ABS(" @@ -492,15 +492,15 @@ def test_giqlnearest_sql_with_max_distance(self, tables_with_peaks_and_genes): 'WHEN peaks."start" < genes."end" ' 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end") ' - 'ELSE (peaks."start" - genes."end") END)) <= 100000\n' + 'THEN (genes."start" - peaks."end" + 1) ' + 'ELSE (peaks."start" - genes."end" + 1) END)) <= 100000\n' " ORDER BY ABS(" 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' 'WHEN peaks."start" < genes."end" ' 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end") ' - 'ELSE (peaks."start" - genes."end") END)\n' + 'THEN (genes."start" - peaks."end" + 1) ' + 'ELSE (peaks."start" - genes."end" + 1) END)\n' " LIMIT 5\n" " )" ) @@ -531,11 +531,11 @@ def test_giqlnearest_sql_stranded(self, tables_with_peaks_and_genes): 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' "THEN CASE WHEN peaks.\"strand\" = '-' " - 'THEN -(genes."start" - peaks."end") ' - 'ELSE (genes."start" - peaks."end") END ' + 'THEN -(genes."start" - peaks."end" + 1) ' + 'ELSE (genes."start" - peaks."end" + 1) END ' "ELSE CASE WHEN peaks.\"strand\" = '-' " - 'THEN -(peaks."start" - genes."end") ' - 'ELSE (peaks."start" - genes."end") END END AS distance\n' + 'THEN -(peaks."start" - genes."end" + 1) ' + 'ELSE (peaks."start" - genes."end" + 1) END END AS distance\n' " FROM genes\n" ' WHERE peaks."chrom" = genes."chrom" ' 'AND peaks."strand" = genes."strand"\n' @@ -548,11 +548,11 @@ def test_giqlnearest_sql_stranded(self, tables_with_peaks_and_genes): 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' "THEN CASE WHEN peaks.\"strand\" = '-' " - 'THEN -(genes."start" - peaks."end") ' - 'ELSE (genes."start" - peaks."end") END ' + 'THEN -(genes."start" - peaks."end" + 1) ' + 'ELSE (genes."start" - peaks."end" + 1) END ' "ELSE CASE WHEN peaks.\"strand\" = '-' " - 'THEN -(peaks."start" - genes."end") ' - 'ELSE (peaks."start" - genes."end") END END)\n' + 'THEN -(peaks."start" - genes."end" + 1) ' + 'ELSE (peaks."start" - genes."end" + 1) END END)\n' " LIMIT 3\n" " )" ) @@ -603,8 +603,8 @@ def test_giqlnearest_sql_signed(self, tables_with_peaks_and_genes): 'WHEN peaks."start" < genes."end" ' 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end") ' - 'ELSE -(peaks."start" - genes."end") END AS distance\n' + 'THEN (genes."start" - peaks."end" + 1) ' + 'ELSE -(peaks."start" - genes."end" + 1) END AS distance\n' " FROM genes\n" ' WHERE peaks."chrom" = genes."chrom"\n' " ORDER BY ABS(" @@ -612,8 +612,8 @@ def test_giqlnearest_sql_signed(self, tables_with_peaks_and_genes): 'WHEN peaks."start" < genes."end" ' 'AND peaks."end" > genes."start" THEN 0 ' 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end") ' - 'ELSE -(peaks."start" - genes."end") END)\n' + 'THEN (genes."start" - peaks."end" + 1) ' + 'ELSE -(peaks."start" - genes."end" + 1) END)\n' " LIMIT 3\n" " )" ) @@ -683,8 +683,8 @@ def test_giqldistance_sql_basic(self, tables_with_two_tables): 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > b."start" ' 'THEN 0 WHEN a."end" <= b."start" ' - 'THEN (b."start" - a."end") ' - 'ELSE (a."start" - b."end") END AS dist ' + 'THEN (b."start" - a."end" + 1) ' + 'ELSE (a."start" - b."end" + 1) END AS dist ' "FROM features_a AS a CROSS JOIN features_b AS b" ) assert output == expected @@ -711,11 +711,11 @@ def test_giqldistance_sql_stranded(self, tables_with_two_tables): 'AND a."end" > b."start" THEN 0 ' 'WHEN a."end" <= b."start" ' "THEN CASE WHEN a.\"strand\" = '-' " - 'THEN -(b."start" - a."end") ' - 'ELSE (b."start" - a."end") END ' + 'THEN -(b."start" - a."end" + 1) ' + 'ELSE (b."start" - a."end" + 1) END ' "ELSE CASE WHEN a.\"strand\" = '-' " - 'THEN -(a."start" - b."end") ' - 'ELSE (a."start" - b."end") END END AS dist ' + 'THEN -(a."start" - b."end" + 1) ' + 'ELSE (a."start" - b."end" + 1) END END AS dist ' "FROM features_a AS a CROSS JOIN features_b AS b" ) assert output == expected @@ -737,8 +737,8 @@ def test_giqldistance_sql_signed(self, tables_with_two_tables): 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > b."start" ' 'THEN 0 WHEN a."end" <= b."start" ' - 'THEN (b."start" - a."end") ' - 'ELSE -(a."start" - b."end") END AS dist ' + 'THEN (b."start" - a."end" + 1) ' + 'ELSE -(a."start" - b."end" + 1) END AS dist ' "FROM features_a AS a CROSS JOIN features_b AS b" ) assert output == expected @@ -766,27 +766,28 @@ def test_giqldistance_sql_stranded_and_signed(self, tables_with_two_tables): 'AND a."end" > b."start" THEN 0 ' 'WHEN a."end" <= b."start" ' "THEN CASE WHEN a.\"strand\" = '-' " - 'THEN -(b."start" - a."end") ' - 'ELSE (b."start" - a."end") END ' + 'THEN -(b."start" - a."end" + 1) ' + 'ELSE (b."start" - a."end" + 1) END ' "ELSE CASE WHEN a.\"strand\" = '-' " - 'THEN (a."start" - b."end") ' - 'ELSE -(a."start" - b."end") END END AS dist ' + 'THEN (a."start" - b."end" + 1) ' + 'ELSE -(a."start" - b."end" + 1) END END AS dist ' "FROM features_a AS a CROSS JOIN features_b AS b" ) assert output == expected - def test_giqldistance_should_not_apply_gap_plus_one_for_closed_intervals( + def test_giqldistance_canonicalizes_closed_ends_apart_from_gap_parity( self, tables_with_closed_intervals ): - """Test DISTANCE on closed-interval tables omits "+1" gap counting. + """Test DISTANCE on closed-interval tables keeps encoding and gap "+1" separate. Given: Two 0-based closed-interval tables and DISTANCE. When: giqldistance_sql is called. Then: - It should canonicalize each table-side end as (end + 1) but omit - any "+1" from the gap branches. + It should canonicalize each table-side end as (end + 1) for the + closed->half-open conversion, distinct from the bedtools-parity + + 1 the gap branches add to the canonical gap. """ # Arrange tables_with_closed_intervals.register( @@ -807,8 +808,8 @@ def test_giqldistance_should_not_apply_gap_plus_one_for_closed_intervals( 'WHEN a."start" < (b."end" + 1) ' 'AND (a."end" + 1) > b."start" THEN 0 ' 'WHEN (a."end" + 1) <= b."start" ' - 'THEN (b."start" - (a."end" + 1)) ' - 'ELSE (a."start" - (b."end" + 1)) END AS dist ' + 'THEN (b."start" - (a."end" + 1) + 1) ' + 'ELSE (a."start" - (b."end" + 1) + 1) END AS dist ' "FROM bed_features AS a CROSS JOIN bed_features_b AS b" ) assert output == expected @@ -898,10 +899,10 @@ def test_giqlnearest_sql_stranded_implicit_reference( assert 'peaks."strand"' in output assert 'genes."strand"' in output - def test_giqlnearest_should_not_apply_gap_plus_one_for_closed_intervals( + def test_giqlnearest_canonicalizes_closed_end_in_wrapper_apart_from_gap_parity( self, ): - """Test NEAREST on a closed-interval target omits "+1" gap counting. + """Test NEAREST on a closed-interval target keeps encoding and gap "+1" separate. Given: A 0-based closed-interval target table and NEAREST. @@ -909,8 +910,8 @@ def test_giqlnearest_should_not_apply_gap_plus_one_for_closed_intervals( The query is transpiled. Then: It should canonicalize the target end as (end + 1) inside the - wrapper CTE while the distance gap branches read the bare canonical - end with no trailing "+ 1". + wrapper CTE, distinct from the bedtools-parity + 1 the distance gap + branches add to the canonical gap. """ # Arrange sql = ( @@ -923,12 +924,36 @@ def test_giqlnearest_should_not_apply_gap_plus_one_for_closed_intervals( # the distance CASE reads the bare canonical columns. output = transpile(sql, tables=[Table("genes_closed", interval_type="closed")]) - # Assert — the wrapper carries (end + 1); the gap branches carry no "+ 1". + # Assert — the wrapper carries the closed->half-open (end + 1); the gap + # branches read the canonical columns and add the bedtools-parity + 1. assert '("end" + 1) AS "end"' in output - assert 'THEN (__giql_canon_0."start" - 2000)' in output - assert 'ELSE (1000 - __giql_canon_0."end")' in output - assert '__giql_canon_0."start" - 2000 + 1' not in output - assert '1000 - __giql_canon_0."end" + 1' not in output + assert 'THEN (__giql_canon_0."start" - 2000 + 1)' in output + assert 'ELSE (1000 - __giql_canon_0."end" + 1)' in output + + def test_giqlnearest_closed_interval_does_not_double_count_plus_one(self): + """Test NEAREST on a closed target never fuses the two +1 terms. + + Given: + A 0-based closed-interval target table and NEAREST. + When: + The query is transpiled. + Then: + It should keep the encoding (end + 1) and the gap-parity + 1 as + separate terms — never collapsing them into a `+ 1 + 1` / + `(end + 1 + 1)` that would double-count the off-by-one. + """ + # Arrange + sql = ( + "SELECT * FROM NEAREST(genes_closed, reference := 'chr1:1000-2000', k := 3)" + ) + + # Act + output = transpile(sql, tables=[Table("genes_closed", interval_type="closed")]) + + # Assert + assert "+ 1 + 1" not in output + assert '("end" + 1 + 1)' not in output + assert '("start" + 1 + 1)' not in output def test_giqldistance_sql_literal_first_arg_error(self, tables_with_two_tables): """ @@ -1607,8 +1632,8 @@ def test_giqldistance_should_canonicalize_table_columns_for_each_convention( expected = ( 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " - f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a}) " - f"ELSE ({start_a} - {end_b}) END AS dist " + f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " + f"ELSE ({start_a} - {end_b} + 1) END AS dist " "FROM dist_a AS a CROSS JOIN dist_b AS b" ) assert output == expected @@ -1642,8 +1667,8 @@ def test_giqldistance_should_canonicalize_each_side_when_conventions_differ( 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > (b."start" - 1) THEN 0 ' 'WHEN a."end" <= (b."start" - 1) ' - 'THEN ((b."start" - 1) - a."end") ' - 'ELSE (a."start" - b."end") END AS dist ' + 'THEN ((b."start" - 1) - a."end" + 1) ' + 'ELSE (a."start" - b."end" + 1) END AS dist ' "FROM bed_a AS a CROSS JOIN vcf_b AS b" ) assert output == expected @@ -1715,9 +1740,9 @@ def test_giqlnearest_should_canonicalize_target_columns_for_each_convention( 'WHEN 1000 < __giql_canon_0."end" AND 2000 > __giql_canon_0."start" THEN 0' ) in output assert ( - 'WHEN 2000 <= __giql_canon_0."start" THEN (__giql_canon_0."start" - 2000)' + 'WHEN 2000 <= __giql_canon_0."start" THEN (__giql_canon_0."start" - 2000 + 1)' ) in output - assert 'ELSE (1000 - __giql_canon_0."end")' in output + assert 'ELSE (1000 - __giql_canon_0."end" + 1)' in output def test_giqlnearest_should_pass_target_columns_through_when_target_is_canonical( self, @@ -1836,5 +1861,5 @@ def test_giqlnearest_should_canonicalize_outer_table_columns_when_reference_is_i 'AND vcf_b."end" > bed_a."start" THEN 0' ) in output assert 'WHEN vcf_b."end" <= bed_a."start"' in output - assert 'THEN (bed_a."start" - vcf_b."end")' in output - assert 'ELSE ((vcf_b."start" - 1) - bed_a."end")' in output + assert 'THEN (bed_a."start" - vcf_b."end" + 1)' in output + assert 'ELSE ((vcf_b."start" - 1) - bed_a."end" + 1)' in output diff --git a/tests/integration/bedtools/test_distance.py b/tests/integration/bedtools/test_distance.py index 4fc53e7..32f2be0 100644 --- a/tests/integration/bedtools/test_distance.py +++ b/tests/integration/bedtools/test_distance.py @@ -5,69 +5,171 @@ closest -d output. """ +import pytest + +from .utils.bedtools_wrapper import closest from .utils.data_models import GenomicInterval +def _oracle_distance(a: GenomicInterval, b: GenomicInterval) -> int: + """Return the bedtools ``closest -d`` distance between a single A and B.""" + return closest([a.to_tuple()], [b.to_tuple()])[0][12] + + def test_distance_non_overlapping(giql_query): """ GIVEN two non-overlapping intervals with a known gap WHEN DISTANCE is computed via GIQL - THEN the distance equals b.start - a.end (half-open arithmetic) + THEN the distance matches live bedtools closest -d """ + a = GenomicInterval("chr1", 100, 200, "a1", 100, "+") + b = GenomicInterval("chr1", 300, 400, "b1", 100, "+") result = giql_query( """ SELECT DISTANCE(a.interval, b.interval) AS dist FROM intervals_a a, intervals_b b """, tables=["intervals_a", "intervals_b"], - intervals_a=[GenomicInterval("chr1", 100, 200, "a1", 100, "+")], - intervals_b=[GenomicInterval("chr1", 300, 400, "b1", 100, "+")], + intervals_a=[a], + intervals_b=[b], ) assert len(result) == 1 - # Half-open distance: b.start - a.end = 300 - 200 = 100 - assert result[0][0] == 100 + assert result[0][0] == _oracle_distance(a, b) def test_distance_overlapping(giql_query): """ GIVEN two overlapping intervals WHEN DISTANCE is computed via GIQL - THEN the distance is 0 + THEN the distance is 0, matching live bedtools closest -d """ + a = GenomicInterval("chr1", 100, 300, "a1", 100, "+") + b = GenomicInterval("chr1", 200, 400, "b1", 100, "+") result = giql_query( """ SELECT DISTANCE(a.interval, b.interval) AS dist FROM intervals_a a, intervals_b b """, tables=["intervals_a", "intervals_b"], - intervals_a=[GenomicInterval("chr1", 100, 300, "a1", 100, "+")], - intervals_b=[GenomicInterval("chr1", 200, 400, "b1", 100, "+")], + intervals_a=[a], + intervals_b=[b], ) assert len(result) == 1 - assert result[0][0] == 0 + assert result[0][0] == _oracle_distance(a, b) def test_distance_adjacent(giql_query): """ GIVEN two adjacent intervals (touching, half-open coordinates) WHEN DISTANCE is computed via GIQL - THEN the distance is 0 (half-open: end of A == start of B means no gap) + THEN the distance is 1, matching live bedtools closest -d for book-ended features """ + a = GenomicInterval("chr1", 100, 200, "a1", 100, "+") + b = GenomicInterval("chr1", 200, 300, "b1", 100, "+") result = giql_query( """ SELECT DISTANCE(a.interval, b.interval) AS dist FROM intervals_a a, intervals_b b """, tables=["intervals_a", "intervals_b"], - intervals_a=[GenomicInterval("chr1", 100, 200, "a1", 100, "+")], - intervals_b=[GenomicInterval("chr1", 200, 300, "b1", 100, "+")], + intervals_a=[a], + intervals_b=[b], + ) + + assert len(result) == 1 + assert result[0][0] == _oracle_distance(a, b) + + +@pytest.mark.parametrize( + ("b_start", "b_end"), + [ + (150, 250), # overlapping + (200, 300), # book-ended downstream + (201, 300), # 1 bp gap + (202, 300), # 2 bp gap + (300, 400), # 100 bp gap + (10200, 10300), # large gap + ], + ids=["overlap", "book_ended", "gap_1bp", "gap_2bp", "gap_100bp", "gap_large"], +) +def test_distance_matches_bedtools_across_gap_sizes(giql_query, b_start, b_end): + """ + GIVEN A chr1:100-200 and B at a range of gaps from overlap to a large gap + WHEN DISTANCE is computed via GIQL + THEN the result equals live bedtools closest -d for every gap, so a future + off-by-one in either direction fails automatically + """ + a = GenomicInterval("chr1", 100, 200, "a1", 100, "+") + b = GenomicInterval("chr1", b_start, b_end, "b1", 100, "+") + result = giql_query( + """ + SELECT DISTANCE(a.interval, b.interval) AS dist + FROM intervals_a a, intervals_b b + """, + tables=["intervals_a", "intervals_b"], + intervals_a=[a], + intervals_b=[b], + ) + + assert len(result) == 1 + assert result[0][0] == _oracle_distance(a, b) + + +def test_distance_book_ended_upstream_matches_bedtools(giql_query): + """ + GIVEN a book-ended pair with B upstream of A (A chr1:200-300, B chr1:100-200) + WHEN DISTANCE is computed via GIQL + THEN the result is 1, matching bedtools, confirming direction symmetry of + the parity +1 + """ + a = GenomicInterval("chr1", 200, 300, "a1", 100, "+") + b = GenomicInterval("chr1", 100, 200, "b1", 100, "+") + result = giql_query( + """ + SELECT DISTANCE(a.interval, b.interval) AS dist + FROM intervals_a a, intervals_b b + """, + tables=["intervals_a", "intervals_b"], + intervals_a=[a], + intervals_b=[b], ) assert len(result) == 1 - # Half-open adjacent: b.start - a.end = 200 - 200 = 0 - assert result[0][0] == 0 + assert result[0][0] == _oracle_distance(a, b) + + +def test_distance_multi_row_cross_join_matches_bedtools(giql_query): + """ + GIVEN two A intervals and two B intervals at mixed gaps in one query + WHEN DISTANCE is computed for every (a, b) pair via a GIQL cross join + THEN each pair's distance equals live bedtools closest -d for that pair, + exercising the multi-row table-scan path + """ + a_rows = [ + GenomicInterval("chr1", 100, 200, "a1", 100, "+"), + GenomicInterval("chr1", 500, 600, "a2", 100, "+"), + ] + b_rows = [ + GenomicInterval("chr1", 200, 300, "b1", 100, "+"), + GenomicInterval("chr1", 800, 900, "b2", 100, "+"), + ] + result = giql_query( + """ + SELECT a.name, b.name, DISTANCE(a.interval, b.interval) AS dist + FROM intervals_a a, intervals_b b + """, + tables=["intervals_a", "intervals_b"], + intervals_a=a_rows, + intervals_b=b_rows, + ) + + by_pair = {(row[0], row[1]): row[2] for row in result} + assert len(by_pair) == 4 + for a in a_rows: + for b in b_rows: + assert by_pair[(a.name, b.name)] == _oracle_distance(a, b) def test_distance_cross_chromosome(giql_query): @@ -96,7 +198,7 @@ def test_distance_signed_downstream(giql_query): """ GIVEN B is downstream of A (B starts after A ends) on + strand WHEN DISTANCE with signed := true is computed - THEN the distance is positive (downstream) + THEN the distance is +101, the gap (100) plus the bedtools parity 1 """ result = giql_query( """ @@ -109,14 +211,14 @@ def test_distance_signed_downstream(giql_query): ) assert len(result) == 1 - assert result[0][0] > 0, f"Expected positive (downstream), got {result[0][0]}" + assert result[0][0] == 101, f"Expected +101 (downstream), got {result[0][0]}" def test_distance_signed_upstream(giql_query): """ GIVEN B is upstream of A (B ends before A starts) on + strand WHEN DISTANCE with signed := true is computed - THEN the distance is negative (upstream) + THEN the distance is -101, the gap (100) plus parity 1, negated for upstream """ result = giql_query( """ @@ -129,7 +231,7 @@ def test_distance_signed_upstream(giql_query): ) assert len(result) == 1 - assert result[0][0] < 0, f"Expected negative (upstream), got {result[0][0]}" + assert result[0][0] == -101, f"Expected -101 (upstream), got {result[0][0]}" def test_distance_stranded_unstranded_input(giql_query): @@ -158,21 +260,23 @@ def test_distance_stranded_same_strand(giql_query): """ GIVEN two non-overlapping intervals both on + strand WHEN DISTANCE with stranded := true is computed - THEN the distance is computed normally (same strand is valid) + THEN the distance matches live bedtools closest -d (same + strand, no flip) """ + a = GenomicInterval("chr1", 100, 200, "a1", 100, "+") + b = GenomicInterval("chr1", 300, 400, "b1", 100, "+") result = giql_query( """ SELECT DISTANCE(a.interval, b.interval, stranded := true) AS dist FROM intervals_a a, intervals_b b """, tables=["intervals_a", "intervals_b"], - intervals_a=[GenomicInterval("chr1", 100, 200, "a1", 100, "+")], - intervals_b=[GenomicInterval("chr1", 300, 400, "b1", 100, "+")], + intervals_a=[a], + intervals_b=[b], ) assert len(result) == 1 - # Both on same + strand, distance should be computed normally - assert result[0][0] == 100, f"Expected 100, got {result[0][0]}" + # Same + strand applies no flip, so the value matches the unsigned oracle. + assert result[0][0] == _oracle_distance(a, b) def test_distance_signed_stranded_minus_strand(giql_query): @@ -197,8 +301,6 @@ def test_distance_signed_stranded_minus_strand(giql_query): ) assert len(result) == 1 - # On - strand with signed+stranded, genomic downstream becomes - # transcript upstream, so sign should be negative - assert result[0][0] < 0, ( - f"Expected negative distance on - strand, got {result[0][0]}" - ) + # On - strand with signed+stranded, genomic downstream becomes transcript + # upstream, so the sign flips: -(gap 100 + parity 1) = -101. + assert result[0][0] == -101, f"Expected -101 on - strand, got {result[0][0]}" diff --git a/tests/integration/bedtools/test_nearest.py b/tests/integration/bedtools/test_nearest.py index 3a91641..4b0ea8d 100644 --- a/tests/integration/bedtools/test_nearest.py +++ b/tests/integration/bedtools/test_nearest.py @@ -45,36 +45,36 @@ def test_nearest_non_overlapping(duckdb_connection): sql = transpile( """ - SELECT a.*, b.* + SELECT a.name, b.name, b.distance FROM intervals_a a CROSS JOIN LATERAL NEAREST( intervals_b, reference := a.interval, k := 1 ) b - ORDER BY a.chrom, a.start """, tables=["intervals_a", "intervals_b"], ) giql_result = duckdb_connection.execute(sql).fetchall() - # Verify row counts match - assert len(giql_result) == len(bedtools_result), ( - f"Row count mismatch: GIQL={len(giql_result)}, bedtools={len(bedtools_result)}" + # Align both sides by A name; the bedtools closest row is BED6+BED6+distance, + # so A name is field 3, B name field 9, and distance field 12. + giql_by_a = {a_name: (b_name, distance) for a_name, b_name, distance in giql_result} + bt_by_a = {row[3]: (row[9], row[12]) for row in bedtools_result} + + assert giql_by_a.keys() == bt_by_a.keys(), ( + f"A coverage mismatch: GIQL={set(giql_by_a)}, bedtools={set(bt_by_a)}" ) - # Verify each A interval found the correct B neighbor - for giql_row, bt_row in zip( - sorted(giql_result, key=lambda r: (r[0], r[1])), - sorted(bedtools_result, key=lambda r: (r[0], r[1])), - ): - # Compare A interval name - assert giql_row[3] == bt_row[3], ( - f"A name mismatch: GIQL={giql_row[3]}, bedtools={bt_row[3]}" + for a_name, (bt_b_name, bt_distance) in bt_by_a.items(): + giql_b_name, giql_distance = giql_by_a[a_name] + # Matched B neighbor must agree. + assert giql_b_name == bt_b_name, ( + f"{a_name} B mismatch: GIQL={giql_b_name}, bedtools={bt_b_name}" ) - # Compare matched B interval name - assert giql_row[9] == bt_row[9], ( - f"B name mismatch: GIQL={giql_row[9]}, bedtools={bt_row[9]}" + # GIQL distance is signed; its magnitude must equal bedtools closest -d. + assert abs(giql_distance) == bt_distance, ( + f"{a_name} distance mismatch: GIQL={giql_distance}, bedtools={bt_distance}" ) @@ -173,11 +173,11 @@ def test_nearest_boundary_cases(giql_query): """ GIVEN adjacent intervals (touching but not overlapping) WHEN NEAREST operator is applied - THEN adjacent interval is reported as nearest (distance = 0) + THEN adjacent interval is reported as nearest (distance = 1, bedtools parity) """ result = giql_query( """ - SELECT a.*, b.* + SELECT b.name, b.distance FROM intervals_a a CROSS JOIN LATERAL NEAREST( intervals_b, @@ -194,8 +194,10 @@ def test_nearest_boundary_cases(giql_query): ) assert len(result) == 1 - # b1 is adjacent (distance 0), b2 is far (distance 300) - assert result[0][9] == "b1" + # b1 is adjacent (book-ended), reported as distance 1 under bedtools parity; + # b2 is far (301). + assert result[0][0] == "b1" + assert result[0][1] == 1 def test_nearest_k_greater_than_one(giql_query): @@ -228,6 +230,51 @@ def test_nearest_k_greater_than_one(giql_query): assert set(b_names) == {"b_far", "b_near", "b_farther"} +def test_nearest_k3_reported_distance_matches_bedtools(duckdb_connection): + """ + GIVEN one query interval and three candidates at distinct (untied) gaps + WHEN NEAREST with k := 3 is applied and the distance column is read + THEN each neighbor's reported distance magnitude equals live bedtools + closest -d -k 3 for the same neighbor + """ + intervals_a = [GenomicInterval("chr1", 200, 300, "a1", 100, "+")] + intervals_b = [ + GenomicInterval("chr1", 310, 350, "b_near", 100, "+"), # gap 10 -> 11 + GenomicInterval("chr1", 400, 450, "b_mid", 100, "+"), # gap 100 -> 101 + GenomicInterval("chr1", 600, 700, "b_far", 100, "+"), # gap 300 -> 301 + ] + + load_intervals(duckdb_connection, "intervals_a", [i.to_tuple() for i in intervals_a]) + load_intervals(duckdb_connection, "intervals_b", [i.to_tuple() for i in intervals_b]) + + bedtools_result = closest( + [i.to_tuple() for i in intervals_a], + [i.to_tuple() for i in intervals_b], + k=3, + ) + oracle_by_name = {row[9]: row[12] for row in bedtools_result} + + sql = transpile( + """ + SELECT b.name, b.distance + FROM intervals_a a + CROSS JOIN LATERAL NEAREST( + intervals_b, + reference := a.interval, + k := 3 + ) b + """, + tables=["intervals_a", "intervals_b"], + ) + giql_result = duckdb_connection.execute(sql).fetchall() + + assert len(giql_result) == 3 + for name, distance in giql_result: + assert abs(distance) == oracle_by_name[name], ( + f"{name}: GIQL={distance}, bedtools={oracle_by_name[name]}" + ) + + def test_nearest_k_exceeds_available(giql_query): """ GIVEN one query interval and only two database intervals @@ -257,13 +304,15 @@ def test_nearest_k_exceeds_available(giql_query): def test_nearest_max_distance(giql_query): """ - GIVEN one query interval, one near and one far database interval + GIVEN candidates whose reported distances straddle max_distance := 50: one + below, one exactly at 50, and one at 51 WHEN NEAREST with max_distance := 50 is applied - THEN only the near interval (within 50bp) is returned + THEN the candidates at 11 and exactly 50 are kept and the one at 51 is + excluded, pinning the inclusive `<= max_distance` boundary on both sides """ result = giql_query( """ - SELECT a.name, b.name + SELECT b.name, b.distance FROM intervals_a a CROSS JOIN LATERAL NEAREST( intervals_b, @@ -275,14 +324,21 @@ def test_nearest_max_distance(giql_query): tables=["intervals_a", "intervals_b"], intervals_a=[GenomicInterval("chr1", 200, 300, "a1", 100, "+")], intervals_b=[ + # gap 10 -> reported 11 (well within) GenomicInterval("chr1", 310, 350, "b_near", 100, "+"), - GenomicInterval("chr1", 500, 600, "b_far", 100, "+"), + # gap 49 -> reported 50 == max_distance (inclusive, kept) + GenomicInterval("chr1", 349, 400, "b_edge", 100, "+"), + # gap 50 -> reported 51 == max_distance + 1 (excluded) + GenomicInterval("chr1", 350, 450, "b_over", 100, "+"), ], ) - b_names = [row[1] for row in result] - # b_near is 10bp away (310 - 300), b_far is 200bp away (500 - 300) - assert b_names == ["b_near"], f"Expected only b_near within 50bp, got {b_names}" + by_name = {row[0]: row[1] for row in result} + assert sorted(by_name) == ["b_edge", "b_near"], ( + f"Expected b_near and b_edge within 50bp, got {sorted(by_name)}" + ) + assert by_name["b_near"] == 11 + assert by_name["b_edge"] == 50 # exactly at the boundary, kept def test_nearest_standalone_literal_reference(giql_query): diff --git a/tests/integration/bedtools/test_nearest_property.py b/tests/integration/bedtools/test_nearest_property.py new file mode 100644 index 0000000..7c14a09 --- /dev/null +++ b/tests/integration/bedtools/test_nearest_property.py @@ -0,0 +1,153 @@ +"""Property-based correctness tests for the NEAREST distance shift (#134). + +These tests use Hypothesis to generate random reference intervals and +candidate sets, then verify three invariants of the bedtools-parity +1 +applied to NEAREST: the reported distance equals the half-open gap plus +one (zero for overlaps), the k-nearest *selection* is unchanged by the +uniform monotonic shift, and ``max_distance`` filters on the shifted +distance. +""" + +import pytest +from hypothesis import HealthCheck +from hypothesis import given +from hypothesis import settings +from hypothesis import strategies as st + +from giql import transpile + +from .utils.data_models import GenomicInterval +from .utils.duckdb_loader import load_intervals + +duckdb = pytest.importorskip("duckdb") + + +def _expected_distance(ref: GenomicInterval, cand: GenomicInterval) -> int: + """Compute the bedtools-parity distance between two half-open intervals. + + Overlapping intervals are 0; otherwise the raw half-open gap plus one. + """ + if ref.start < cand.end and ref.end > cand.start: + return 0 + if cand.start >= ref.end: + return cand.start - ref.end + 1 + return ref.start - cand.end + 1 + + +@st.composite +def _candidates(draw, min_size=1, max_size=20): + """Generate a list of same-chromosome candidate intervals with unique names.""" + n = draw(st.integers(min_value=min_size, max_value=max_size)) + candidates = [] + for i in range(n): + start = draw(st.integers(min_value=0, max_value=120_000)) + length = draw(st.integers(min_value=1, max_value=5_000)) + candidates.append( + GenomicInterval("chr1", start, start + length, f"c{i}", 100, "+") + ) + return candidates + + +@st.composite +def _reference(draw): + """Generate a single canonical (0-based half-open) reference interval.""" + start = draw(st.integers(min_value=0, max_value=100_000)) + length = draw(st.integers(min_value=1, max_value=5_000)) + return GenomicInterval("chr1", start, start + length, "ref", 100, "+") + + +def _run_nearest(reference, candidates, k, max_distance=None): + """Run standalone NEAREST in DuckDB and return [(name, distance), ...].""" + conn = duckdb.connect(":memory:") + try: + load_intervals(conn, "candidates", [c.to_tuple() for c in candidates]) + max_clause = "" if max_distance is None else f", max_distance := {max_distance}" + sql = transpile( + f""" + SELECT name, distance + FROM NEAREST( + candidates, + reference := 'chr1:{reference.start}-{reference.end}', + k := {k}{max_clause} + ) + """, + tables=["candidates"], + ) + return conn.execute(sql).fetchall() + finally: + conn.close() + + +@given(reference=_reference(), candidates=_candidates()) +@settings( + max_examples=50, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +def test_nearest_reported_distance_equals_gap_plus_one(reference, candidates): + """ + GIVEN a random reference interval and a random candidate set + WHEN NEAREST returns every candidate with its distance + THEN each reported distance equals the half-open gap + 1 (0 for overlaps) + """ + by_name = {c.name: c for c in candidates} + + rows = _run_nearest(reference, candidates, k=len(candidates)) + + for name, distance in rows: + assert distance == _expected_distance(reference, by_name[name]), ( + f"{name}: reported {distance}, expected " + f"{_expected_distance(reference, by_name[name])}" + ) + + +@given(data=st.data(), reference=_reference(), candidates=_candidates()) +@settings( + max_examples=50, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +def test_nearest_k_selection_is_unchanged_by_the_shift(data, reference, candidates): + """ + GIVEN a random reference, candidate set, and k + WHEN NEAREST returns the k nearest candidates + THEN the multiset of reported distances equals the k smallest distances by + raw half-open gap, so the uniform +1 does not change which candidates + are selected + """ + k = data.draw(st.integers(min_value=1, max_value=len(candidates))) + expected_sorted = sorted(_expected_distance(reference, c) for c in candidates) + + rows = _run_nearest(reference, candidates, k=k) + + reported_sorted = sorted(distance for _, distance in rows) + assert reported_sorted == expected_sorted[:k] + + +@given( + data=st.data(), + reference=_reference(), + candidates=_candidates(), +) +@settings( + max_examples=50, + deadline=None, + suppress_health_check=[HealthCheck.too_slow], +) +def test_nearest_max_distance_filters_on_the_shifted_distance( + data, reference, candidates +): + """ + GIVEN a random reference, candidate set, and threshold + WHEN NEAREST is run with that max_distance over all candidates + THEN the returned candidates are exactly those whose gap + 1 is <= the + threshold, so no candidate at threshold + 1 ever survives + """ + expected = {c.name: _expected_distance(reference, c) for c in candidates} + threshold = data.draw(st.integers(min_value=0, max_value=max(expected.values()) + 5)) + + rows = _run_nearest(reference, candidates, k=len(candidates), max_distance=threshold) + + returned = {name for name, _ in rows} + survivors = {name for name, dist in expected.items() if dist <= threshold} + assert returned == survivors diff --git a/tests/integration/coordinate_space/test_distance_coordinate_space.py b/tests/integration/coordinate_space/test_distance_coordinate_space.py index 0b08cf4..fb464f7 100644 --- a/tests/integration/coordinate_space/test_distance_coordinate_space.py +++ b/tests/integration/coordinate_space/test_distance_coordinate_space.py @@ -31,7 +31,7 @@ class TestDistanceCoordinateSpace: def test_distance_returns_canonical_gap_when_both_sides_share_convention( self, giql_query, coordinate_system, interval_type ): - """Test DISTANCE returns 100 for a 100 bp gap in any same-convention encoding. + """Test DISTANCE returns 101 for a 100 bp gap in any same-convention encoding. Given: Two non-overlapping intervals (canonical [100, 200) and @@ -39,8 +39,9 @@ def test_distance_returns_canonical_gap_when_both_sides_share_convention( When: DISTANCE(a.interval, b.interval) is executed. Then: - It should return 100, matching the canonical 0-based half-open - gap, regardless of how the intervals are stored. + It should return 101 (the canonical 0-based half-open gap of 100 + plus the bedtools closest -d parity + 1), regardless of how the + intervals are stored. """ # Arrange s_a, e_a = encode(100, 200, coordinate_system, interval_type) @@ -62,7 +63,7 @@ def test_distance_returns_canonical_gap_when_both_sides_share_convention( ) # Assert - assert result == [(100,)] + assert result == [(101,)] @pytest.mark.parametrize( ("coordinate_system", "interval_type"), @@ -109,10 +110,10 @@ def test_distance_returns_zero_when_intervals_overlap( CONVENTIONS, ids=lambda v: str(v), ) - def test_distance_returns_zero_for_adjacent_half_open_boundary( + def test_distance_returns_one_for_adjacent_half_open_boundary( self, giql_query, coordinate_system, interval_type ): - """Test DISTANCE returns 0 when intervals touch at the half-open boundary. + """Test DISTANCE returns 1 when intervals touch at the half-open boundary. Given: Canonical adjacent intervals [100, 200) and [200, 300) @@ -120,7 +121,8 @@ def test_distance_returns_zero_for_adjacent_half_open_boundary( When: DISTANCE is executed. Then: - It should return 0 regardless of encoding. + It should return 1 (bedtools closest -d parity for book-ended + features) regardless of encoding. """ # Arrange s_a, e_a = encode(100, 200, coordinate_system, interval_type) @@ -142,7 +144,7 @@ def test_distance_returns_zero_for_adjacent_half_open_boundary( ) # Assert - assert result == [(0,)] + assert result == [(1,)] @pytest.mark.parametrize( ("conv_a", "conv_b"), @@ -152,7 +154,7 @@ def test_distance_returns_zero_for_adjacent_half_open_boundary( def test_distance_is_invariant_under_mixed_conventions( self, giql_query, conv_a, conv_b ): - """Test DISTANCE returns 100 for any pair of (convention_a, convention_b). + """Test DISTANCE returns 101 for any pair of (convention_a, convention_b). Given: Canonical intervals [100, 200) and [300, 400) encoded in @@ -160,9 +162,9 @@ def test_distance_is_invariant_under_mixed_conventions( When: DISTANCE is executed. Then: - It should return 100 for every (convention_a, convention_b) - pair, demonstrating that each side is canonicalized - independently. + It should return 101 (the 100 bp half-open gap plus the bedtools + parity + 1) for every (convention_a, convention_b) pair, + demonstrating that each side is canonicalized independently. """ # Arrange s_a, e_a = encode(100, 200, *conv_a) @@ -184,7 +186,7 @@ def test_distance_is_invariant_under_mixed_conventions( ) # Assert - assert result == [(100,)] + assert result == [(101,)] @pytest.mark.parametrize( ("coordinate_system", "interval_type"), @@ -234,7 +236,7 @@ def test_distance_returns_null_for_cross_chromosome_pair( def test_signed_distance_returns_positive_when_b_is_downstream( self, giql_query, coordinate_system, interval_type ): - """Test signed DISTANCE returns +100 when B is downstream in any encoding. + """Test signed DISTANCE returns +101 when B is downstream in any encoding. Given: Canonical A=[100, 200) and B=[300, 400) re-encoded under the @@ -242,7 +244,8 @@ def test_signed_distance_returns_positive_when_b_is_downstream( When: DISTANCE runs. Then: - It should return +100 regardless of encoding. + It should return +101 (100 bp gap + bedtools parity 1) regardless + of encoding. """ # Arrange s_a, e_a = encode(100, 200, coordinate_system, interval_type) @@ -264,7 +267,7 @@ def test_signed_distance_returns_positive_when_b_is_downstream( ) # Assert - assert result == [(100,)] + assert result == [(101,)] @pytest.mark.parametrize( ("coordinate_system", "interval_type"), @@ -274,7 +277,7 @@ def test_signed_distance_returns_positive_when_b_is_downstream( def test_signed_distance_returns_negative_when_b_is_upstream( self, giql_query, coordinate_system, interval_type ): - """Test signed DISTANCE returns -100 when B is upstream in any encoding. + """Test signed DISTANCE returns -101 when B is upstream in any encoding. Given: Canonical A=[300, 400) and B=[100, 200) re-encoded under the @@ -282,7 +285,8 @@ def test_signed_distance_returns_negative_when_b_is_upstream( When: DISTANCE runs. Then: - It should return -100 regardless of encoding. + It should return -101 (100 bp gap + bedtools parity 1, negated) + regardless of encoding. """ # Arrange s_a, e_a = encode(300, 400, coordinate_system, interval_type) @@ -304,4 +308,169 @@ def test_signed_distance_returns_negative_when_b_is_upstream( ) # Assert - assert result == [(-100,)] + assert result == [(-101,)] + + @pytest.mark.parametrize( + ("conv_a", "conv_b"), + [(a, b) for a in CONVENTIONS for b in CONVENTIONS], + ids=lambda v: str(v), + ) + def test_distance_book_ended_is_one_under_mixed_conventions( + self, giql_query, conv_a, conv_b + ): + """Test DISTANCE returns 1 for a book-ended pair for any convention pair. + + Given: + Canonical book-ended intervals [100, 200) and [200, 300) encoded in + convention_a on table A and convention_b on table B. + When: + DISTANCE is executed. + Then: + It should return 1 (bedtools parity for touching features) for every + (convention_a, convention_b) pair, proving the +1 lands on the + canonical gap regardless of how each side is stored. + """ + # Arrange + s_a, e_a = encode(100, 200, *conv_a) + s_b, e_b = encode(200, 300, *conv_b) + tables = [ + make_table("intervals_a", *conv_a), + make_table("intervals_b", *conv_b), + ] + + # Act + result = giql_query( + """ + SELECT DISTANCE(a.interval, b.interval) AS dist + FROM intervals_a a, intervals_b b + """, + tables=tables, + intervals_a=[_row("chr1", s_a, e_a, "a1")], + intervals_b=[_row("chr1", s_b, e_b, "b1")], + ) + + # Assert + assert result == [(1,)] + + @pytest.mark.parametrize( + ("coordinate_system", "interval_type"), + CONVENTIONS, + ids=lambda v: str(v), + ) + def test_distance_one_bp_gap_returns_two_in_any_encoding( + self, giql_query, coordinate_system, interval_type + ): + """Test DISTANCE returns 2 for a 1 bp gap in any same-convention encoding. + + Given: + Canonical intervals [100, 200) and [201, 300) (a 1 bp half-open + gap) re-encoded under the same convention on both sides. + When: + DISTANCE is executed. + Then: + It should return 2 (half-open gap 1 + bedtools parity 1) regardless + of encoding. + """ + # Arrange + s_a, e_a = encode(100, 200, coordinate_system, interval_type) + s_b, e_b = encode(201, 300, coordinate_system, interval_type) + tables = [ + make_table("intervals_a", coordinate_system, interval_type), + make_table("intervals_b", coordinate_system, interval_type), + ] + + # Act + result = giql_query( + """ + SELECT DISTANCE(a.interval, b.interval) AS dist + FROM intervals_a a, intervals_b b + """, + tables=tables, + intervals_a=[_row("chr1", s_a, e_a, "a1")], + intervals_b=[_row("chr1", s_b, e_b, "b1")], + ) + + # Assert + assert result == [(2,)] + + @pytest.mark.parametrize( + ("conv_a", "conv_b"), + [(a, b) for a in CONVENTIONS for b in CONVENTIONS], + ids=lambda v: str(v), + ) + def test_distance_one_bp_gap_is_two_under_mixed_conventions( + self, giql_query, conv_a, conv_b + ): + """Test DISTANCE returns 2 for a 1 bp gap for any convention pair. + + Given: + Canonical intervals [100, 200) and [201, 300) (a 1 bp half-open + gap) encoded in convention_a on table A and convention_b on table B. + When: + DISTANCE is executed. + Then: + It should return 2 for every (convention_a, convention_b) pair, the + tightest parity check across independent per-side canonicalization. + """ + # Arrange + s_a, e_a = encode(100, 200, *conv_a) + s_b, e_b = encode(201, 300, *conv_b) + tables = [ + make_table("intervals_a", *conv_a), + make_table("intervals_b", *conv_b), + ] + + # Act + result = giql_query( + """ + SELECT DISTANCE(a.interval, b.interval) AS dist + FROM intervals_a a, intervals_b b + """, + tables=tables, + intervals_a=[_row("chr1", s_a, e_a, "a1")], + intervals_b=[_row("chr1", s_b, e_b, "b1")], + ) + + # Assert + assert result == [(2,)] + + @pytest.mark.parametrize( + ("coordinate_system", "interval_type"), + CONVENTIONS, + ids=lambda v: str(v), + ) + def test_stranded_distance_book_ended_is_one_in_any_encoding( + self, giql_query, coordinate_system, interval_type + ): + """Test stranded DISTANCE returns 1 for a book-ended pair in any encoding. + + Given: + Canonical book-ended intervals [100, 200) and [200, 300), both on + '+' strand, re-encoded under the same convention on both sides. + When: + DISTANCE with stranded := true is executed. + Then: + It should return 1 regardless of encoding, exercising the parity +1 + in the stranded branch under coordinate canonicalization. + """ + # Arrange + s_a, e_a = encode(100, 200, coordinate_system, interval_type) + s_b, e_b = encode(200, 300, coordinate_system, interval_type) + tables = [ + make_table("intervals_a", coordinate_system, interval_type), + make_table("intervals_b", coordinate_system, interval_type), + ] + + # Act + result = giql_query( + """ + SELECT DISTANCE(a.interval, b.interval, stranded := true) AS dist + FROM intervals_a a, intervals_b b + """, + tables=tables, + intervals_a=[_row("chr1", s_a, e_a, "a1")], + intervals_b=[_row("chr1", s_b, e_b, "b1")], + ) + + # Assert + assert result == [(1,)] diff --git a/tests/integration/coordinate_space/test_nearest_coordinate_space.py b/tests/integration/coordinate_space/test_nearest_coordinate_space.py index 5d6933d..5ac9d8c 100644 --- a/tests/integration/coordinate_space/test_nearest_coordinate_space.py +++ b/tests/integration/coordinate_space/test_nearest_coordinate_space.py @@ -154,9 +154,10 @@ def test_correlated_nearest_max_distance_filters_by_canonical_gap( When: NEAREST runs via LATERAL join. Then: - It should return only ``b_near`` (10 bp) and ``b_far`` (50 bp, - inclusive boundary) regardless of encoding; ``b_mid`` (200 bp) - should be excluded. + It should return only ``b_near`` (half-open gap 10 -> reported 11) + regardless of encoding; under bedtools parity ``b_far`` reports 51 + (half-open gap 50 + 1), which exceeds ``max_distance := 50``, so it + joins ``b_mid`` (201) in being excluded. """ # Arrange tables = [ @@ -184,7 +185,7 @@ def test_correlated_nearest_max_distance_filters_by_canonical_gap( ) # Assert - assert sorted(row[0] for row in result) == ["b_far", "b_near"] + assert sorted(row[0] for row in result) == ["b_near"] @pytest.mark.parametrize( ("coordinate_system", "interval_type"), @@ -204,9 +205,9 @@ def test_standalone_nearest_with_literal_reference_resolves_across_encodings( ``SELECT * FROM NEAREST(intervals_b, reference := ..., k := 2)`` runs in standalone mode. Then: - It should return ``b_near`` (10 bp) and ``b_far`` (200 bp) - regardless of target encoding; ``b_mid`` (140 bp) is the - second-nearest, so the expected pair is ``{b_near, b_mid}``. + It should return ``b_near`` (touching, reported 1) and ``b_mid`` + (reported 141) regardless of target encoding; ``b_far`` (reported + 201) is farther, so the expected pair is ``{b_near, b_mid}``. """ # Arrange tables = [make_table("intervals_b", coordinate_system, interval_type)] @@ -227,8 +228,8 @@ def test_standalone_nearest_with_literal_reference_resolves_across_encodings( ) # Assert - # Distances from [350, 360): b_near [310,350) -> 0 (touching), - # b_mid [500,600) -> 140, b_far [100,150) -> 200. + # Distances from [350, 360) with bedtools parity: b_near [310,350) -> + # 1 (touching), b_mid [500,600) -> 141, b_far [100,150) -> 201. assert sorted(row[0] for row in result) == ["b_mid", "b_near"] @pytest.mark.parametrize( @@ -285,13 +286,13 @@ def test_correlated_nearest_is_invariant_under_mixed_conventions( def test_correlated_nearest_picks_adjacent_neighbor_when_touching_half_open_boundary( self, giql_query, coordinate_system, interval_type ): - """Test correlated NEAREST picks the touching candidate (gap 0) in any encoding. + """Test correlated NEAREST picks the touching candidate (reported 1) in any encoding. Given: Canonical A=[100, 200) with two B candidates -- b_adjacent at - canonical [200, 300) (touching half-open boundary, gap 0) and - b_far at [500, 600) -- re-encoded under the same convention on - both sides. + canonical [200, 300) (touching half-open boundary, reported + distance 1 under bedtools parity) and b_far at [500, 600) -- + re-encoded under the same convention on both sides. When: NEAREST k=1 runs via LATERAL join. Then: @@ -302,9 +303,7 @@ def test_correlated_nearest_picks_adjacent_neighbor_when_touching_half_open_boun make_table("intervals_a", coordinate_system, interval_type), make_table("intervals_b", coordinate_system, interval_type), ] - intervals_a = _encode_rows( - [(100, 200, "a1")], coordinate_system, interval_type - ) + intervals_a = _encode_rows([(100, 200, "a1")], coordinate_system, interval_type) intervals_b = _encode_rows( [(200, 300, "b_adjacent"), (500, 600, "b_far")], coordinate_system, diff --git a/tests/test_distance_transpilation.py b/tests/test_distance_transpilation.py index 08efe2a..944a3db 100644 --- a/tests/test_distance_transpilation.py +++ b/tests/test_distance_transpilation.py @@ -45,7 +45,7 @@ def test_distance_transpilation_duckdb(self): output = _generate(sql) - expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end") ELSE (a."start" - b."end") END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" + expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" assert output == expected, f"Expected:\n{expected}\n\nGot:\n{output}" @@ -62,7 +62,7 @@ def test_distance_transpilation_sqlite(self): output = _generate(sql) - expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end") ELSE (a."start" - b."end") END AS dist FROM features_a AS a, features_b AS b""" + expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a, features_b AS b""" assert output == expected, f"Expected:\n{expected}\n\nGot:\n{output}" @@ -79,7 +79,7 @@ def test_distance_transpilation_postgres(self): output = _generate(sql) - expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end") ELSE (a."start" - b."end") END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" + expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" assert output == expected, f"Expected:\n{expected}\n\nGot:\n{output}" @@ -121,8 +121,8 @@ def test_distance_transpilation_signed_duckdb(self): expected = ( 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 ' - 'WHEN a."end" <= b."start" THEN (b."start" - a."end") ' - 'ELSE -(a."start" - b."end") END AS dist ' + 'WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ' + 'ELSE -(a."start" - b."end" + 1) END AS dist ' "FROM features_a AS a CROSS JOIN features_b AS b" ) diff --git a/tests/test_distance_udf.py b/tests/test_distance_udf.py index 8fa03d8..65f9c19 100644 --- a/tests/test_distance_udf.py +++ b/tests/test_distance_udf.py @@ -30,6 +30,15 @@ def _generate(sql: str) -> str: return BaseGIQLGenerator().generate(ast) +def _run(sql: str): + """Generate, execute against in-memory DuckDB, and return the first column.""" + conn = duckdb.connect(":memory:") + try: + return conn.execute(_generate(sql)).fetchone()[0] + finally: + conn.close() + + class TestDistanceCalculation: """Unit tests for basic distance calculation logic.""" @@ -73,9 +82,9 @@ def test_non_overlapping_intervals_return_positive_distance(self): WHEN DISTANCE() is calculated between them THEN the distance should be a positive integer (gap size) """ - # Interval A: chr1:100-200 - # Interval B: chr1:300-400 - # Gap: 300 - 200 = 100 base pairs + # Arrange + # Interval A: chr1:100-200, Interval B: chr1:300-400 + # Half-open gap: 300 - 200 = 100; bedtools closest -d parity adds 1 -> 101 sql = """ SELECT DISTANCE(a.interval, b.interval) as distance @@ -85,15 +94,11 @@ def test_non_overlapping_intervals_return_positive_distance(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end) b """ - output_sql = _generate(sql) - - conn = duckdb.connect(":memory:") - result = conn.execute(output_sql).fetchone() + # Act + result = _run(sql) - # Gap distance should be 100 - assert result[0] == 100, f"Expected distance 100, got {result[0]}" - - conn.close() + # Assert + assert result == 101, f"Expected distance 101, got {result}" def test_different_chromosomes_return_null(self): """ @@ -122,14 +127,15 @@ def test_different_chromosomes_return_null(self): conn.close() - def test_adjacent_bookended_intervals_return_zero(self): + def test_adjacent_bookended_intervals_return_one(self): """ GIVEN two adjacent intervals where end_a == start_b (bookended) WHEN DISTANCE() is calculated between them - THEN the distance should be 0 (following bedtools convention) + THEN the distance should be 1 (bedtools closest -d reports 1 for + book-ended features) """ - # Interval A: chr1:100-200 - # Interval B: chr1:200-300 (starts exactly where A ends) + # Arrange + # Interval A: chr1:100-200, Interval B: chr1:200-300 (starts where A ends) sql = """ SELECT DISTANCE(a.interval, b.interval) as distance @@ -139,17 +145,11 @@ def test_adjacent_bookended_intervals_return_zero(self): (SELECT 'chr1' as chrom, 200 as start, 300 as end) b """ - output_sql = _generate(sql) - - conn = duckdb.connect(":memory:") - result = conn.execute(output_sql).fetchone() - - # Bookended intervals should return distance = 0 - assert result[0] == 0, ( - f"Expected distance 0 for bookended intervals, got {result[0]}" - ) + # Act + result = _run(sql) - conn.close() + # Assert + assert result == 1, f"Expected distance 1 for bookended intervals, got {result}" def test_zero_width_intervals_point_features(self): """ @@ -157,9 +157,9 @@ def test_zero_width_intervals_point_features(self): WHEN DISTANCE() is calculated THEN the distance should be calculated correctly """ - # Point feature at chr1:150 (start=150, end=150) - # Interval at chr1:300-400 - # Distance: 300 - 150 = 150 + # Arrange + # Point feature at chr1:150 (start=150, end=150), interval at chr1:300-400 + # Half-open gap: 300 - 150 = 150; bedtools parity adds 1 -> 151 sql = """ SELECT DISTANCE(a.interval, b.interval) as distance @@ -169,15 +169,144 @@ def test_zero_width_intervals_point_features(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end) b """ - output_sql = _generate(sql) + # Act + result = _run(sql) - conn = duckdb.connect(":memory:") - result = conn.execute(output_sql).fetchone() + # Assert + assert result == 151, f"Expected distance 151, got {result}" - # Distance should be 150 - assert result[0] == 150, f"Expected distance 150, got {result[0]}" + def test_one_base_gap_returns_two(self): + """ + GIVEN two intervals separated by a 1 bp half-open gap (A chr1:100-200, + B chr1:201-300) + WHEN DISTANCE() is calculated between them + THEN the distance should be 2 (half-open gap 1 + bedtools parity 1) + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval) as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end) a + CROSS JOIN (SELECT 'chr1' as chrom, 201 as start, 300 as end) b + """ - conn.close() + # Act + result = _run(sql) + + # Assert + assert result == 2, f"Expected distance 2, got {result}" + + @pytest.mark.parametrize( + ("b_start", "expected"), + [(199, 0), (200, 1), (201, 2), (202, 3)], + ) + def test_distance_increments_by_one_across_the_overlap_boundary( + self, b_start, expected + ): + """ + GIVEN A chr1:100-200 and B starting at 199, 200, 201, or 202 (1 bp + overlap, book-ended, 1 bp gap, 2 bp gap) + WHEN DISTANCE() is calculated between them + THEN the distance steps 0, 1, 2, 3 as the gap opens, pinning the + overlap -> book-ended -> gap transition + """ + # Arrange + sql = f""" + SELECT DISTANCE(a.interval, b.interval) as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end) a + CROSS JOIN (SELECT 'chr1' as chrom, {b_start} as start, 300 as end) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == expected, f"Expected distance {expected}, got {result}" + + def test_single_base_overlap_returns_zero(self): + """ + GIVEN two intervals sharing exactly 1 bp (A chr1:100-200, B chr1:199-300) + WHEN DISTANCE() is calculated between them + THEN the distance should be 0, distinguishing a minimal overlap from a + book-ended pair + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval) as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end) a + CROSS JOIN (SELECT 'chr1' as chrom, 199 as start, 300 as end) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == 0, f"Expected distance 0, got {result}" + + def test_upstream_operand_order_returns_positive_distance(self): + """ + GIVEN A downstream of B (A chr1:300-400, B chr1:100-200), unsigned + WHEN DISTANCE() is calculated between them + THEN the distance should be 101, proving the parity +1 is applied + symmetrically on the upstream (ELSE) branch as well + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval) as distance + FROM (SELECT 'chr1' as chrom, 300 as start, 400 as end) a + CROSS JOIN (SELECT 'chr1' as chrom, 100 as start, 200 as end) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == 101, f"Expected distance 101, got {result}" + + +class TestSignedDistance: + """Unit tests for signed (non-stranded) distance calculation.""" + + def test_signed_book_ended_downstream_returns_positive_one(self): + """ + GIVEN a book-ended pair with B downstream of A (A chr1:100-200, + B chr1:200-300) + WHEN DISTANCE() is calculated with signed := true + THEN the distance should be +1, the issue's named book-ended sign case + (sign applied after the parity +1) + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, signed := true) as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end) a + CROSS JOIN (SELECT 'chr1' as chrom, 200 as start, 300 as end) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == 1, f"Expected distance 1, got {result}" + + def test_signed_book_ended_upstream_returns_negative_one(self): + """ + GIVEN a book-ended pair with B upstream of A (A chr1:200-300, + B chr1:100-200) + WHEN DISTANCE() is calculated with signed := true + THEN the distance should be -1, proving the sign wraps the post-parity + magnitude on the upstream branch + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, signed := true) as distance + FROM (SELECT 'chr1' as chrom, 200 as start, 300 as end) a + CROSS JOIN (SELECT 'chr1' as chrom, 100 as start, 200 as end) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == -1, f"Expected distance -1, got {result}" class TestStrandedDistance: @@ -189,6 +318,7 @@ def test_stranded_same_strand_plus(self): WHEN DISTANCE() is calculated with stranded := true THEN the distance should be calculated normally (positive value) """ + # Arrange sql = """ SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance @@ -198,15 +328,12 @@ def test_stranded_same_strand_plus(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) b """ - output_sql = _generate(sql) - - conn = duckdb.connect(":memory:") - result = conn.execute(output_sql).fetchone() - - # Gap distance should be 100 (positive, since strand is '+') - assert result[0] == 100, f"Expected distance 100, got {result[0]}" + # Act + result = _run(sql) - conn.close() + # Assert + # Gap distance should be 101 (positive, since strand is '+') + assert result == 101, f"Expected distance 101, got {result}" def test_stranded_same_strand_minus(self): """ @@ -214,6 +341,7 @@ def test_stranded_same_strand_minus(self): WHEN DISTANCE() is calculated with stranded := true THEN the distance should be negative (multiplied by -1) """ + # Arrange sql = """ SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance @@ -223,15 +351,12 @@ def test_stranded_same_strand_minus(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '-' as strand) b """ - output_sql = _generate(sql) - - conn = duckdb.connect(":memory:") - result = conn.execute(output_sql).fetchone() - - # Gap distance should be -100 (negative, since first interval strand is '-') - assert result[0] == -100, f"Expected distance -100, got {result[0]}" + # Act + result = _run(sql) - conn.close() + # Assert + # Gap distance should be -101 (negative, since first interval strand is '-') + assert result == -101, f"Expected distance -101, got {result}" def test_stranded_different_strands_calculates_distance(self): """ @@ -239,6 +364,7 @@ def test_stranded_different_strands_calculates_distance(self): WHEN DISTANCE() is calculated with stranded := true THEN the distance should be calculated normally (positive, since first interval is '+') """ + # Arrange sql = """ SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance @@ -248,15 +374,12 @@ def test_stranded_different_strands_calculates_distance(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '-' as strand) b """ - output_sql = _generate(sql) - - conn = duckdb.connect(":memory:") - result = conn.execute(output_sql).fetchone() + # Act + result = _run(sql) - # Different strands should still calculate distance, sign based on first interval - assert result[0] == 100, f"Expected distance 100, got {result[0]}" - - conn.close() + # Assert + # Different strands still calculate distance, sign based on first interval + assert result == 101, f"Expected distance 101, got {result}" def test_stranded_different_strands_minus_first(self): """ @@ -264,6 +387,7 @@ def test_stranded_different_strands_minus_first(self): WHEN DISTANCE() is calculated with stranded := true THEN the distance should be negative (based on first interval's strand) """ + # Arrange sql = """ SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance @@ -273,15 +397,12 @@ def test_stranded_different_strands_minus_first(self): (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) b """ - output_sql = _generate(sql) - - conn = duckdb.connect(":memory:") - result = conn.execute(output_sql).fetchone() + # Act + result = _run(sql) + # Assert # Distance should be negative since first interval is '-' - assert result[0] == -100, f"Expected distance -100, got {result[0]}" - - conn.close() + assert result == -101, f"Expected distance -101, got {result}" def test_stranded_dot_strand_returns_null(self): """ @@ -384,3 +505,194 @@ def test_stranded_overlapping_intervals_minus_strand(self): ) conn.close() + + def test_stranded_book_ended_plus_strand_returns_one(self): + """ + GIVEN a book-ended pair both on '+' strand (A chr1:100-200, + B chr1:200-300) + WHEN DISTANCE() is calculated with stranded := true + THEN the distance should be 1 (parity +1 reaches the stranded branch; + '+' applies no flip) + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end, '+' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 200 as start, 300 as end, '+' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == 1, f"Expected distance 1, got {result}" + + def test_stranded_book_ended_minus_strand_returns_negative_one(self): + """ + GIVEN a book-ended pair both on '-' strand (A chr1:100-200, + B chr1:200-300) + WHEN DISTANCE() is calculated with stranded := true + THEN the distance should be -1, the '-' strand flipping the sign of the + post-parity magnitude + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end, '-' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 200 as start, 300 as end, '-' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == -1, f"Expected distance -1, got {result}" + + def test_stranded_upstream_plus_strand_returns_positive(self): + """ + GIVEN B upstream of A both on '+' strand (A chr1:300-400, B chr1:100-200) + WHEN DISTANCE() is calculated with stranded := true + THEN the distance should be 101, exercising the stranded upstream (ELSE) + branch that the existing stranded tests never reach + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance + FROM (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 100 as start, 200 as end, '+' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == 101, f"Expected distance 101, got {result}" + + def test_stranded_upstream_minus_strand_returns_negative(self): + """ + GIVEN B upstream of A both on '-' strand (A chr1:300-400, B chr1:100-200) + WHEN DISTANCE() is calculated with stranded := true + THEN the distance should be -101, the '-' flip applied to the upstream + branch's post-parity magnitude + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance + FROM (SELECT 'chr1' as chrom, 300 as start, 400 as end, '-' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 100 as start, 200 as end, '-' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == -101, f"Expected distance -101, got {result}" + + def test_stranded_question_strand_returns_null(self): + """ + GIVEN one interval with '?' strand (A chr1:100-200 '?', B chr1:300-400 '+') + WHEN DISTANCE() is calculated with stranded := true + THEN the distance should be NULL, confirming the parity +1 did not + perturb the '?' NULL short-circuit + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true) as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end, '?' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result is None, f"Expected NULL for '?' strand, got {result}" + + +class TestStrandedSignedDistance: + """Unit tests for the combined stranded + signed distance calculation.""" + + def test_stranded_signed_upstream_plus_strand_returns_negative(self): + """ + GIVEN B upstream of A both on '+' strand (A chr1:300-400, B chr1:100-200) + WHEN DISTANCE() is calculated with stranded := true, signed := true + THEN the distance should be -101, the upstream-branch sign that DIVERGES + from the stranded-only variant ('+' -> -(magnitude+1)) + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true, signed := true) + as distance + FROM (SELECT 'chr1' as chrom, 300 as start, 400 as end, '+' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 100 as start, 200 as end, '+' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == -101, f"Expected distance -101, got {result}" + + def test_stranded_signed_upstream_minus_strand_returns_positive(self): + """ + GIVEN B upstream of A both on '-' strand (A chr1:300-400, B chr1:100-200) + WHEN DISTANCE() is calculated with stranded := true, signed := true + THEN the distance should be +101, the only ELSE-branch case where the + sign is +(magnitude+1) ('-' strand double-flip) + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true, signed := true) + as distance + FROM (SELECT 'chr1' as chrom, 300 as start, 400 as end, '-' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 100 as start, 200 as end, '-' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == 101, f"Expected distance 101, got {result}" + + def test_stranded_signed_book_ended_minus_strand_returns_negative_one(self): + """ + GIVEN a book-ended pair both on '-' strand, B downstream (A chr1:100-200, + B chr1:200-300) + WHEN DISTANCE() is calculated with stranded := true, signed := true + THEN the distance should be -1, pinning the sign at the smallest + (book-ended) post-parity magnitude + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true, signed := true) + as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end, '-' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 200 as start, 300 as end, '-' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result == -1, f"Expected distance -1, got {result}" + + def test_stranded_signed_null_strand_returns_null(self): + """ + GIVEN one interval with '.' strand (A chr1:100-200 '.', B chr1:200-300 '+') + WHEN DISTANCE() is calculated with stranded := true, signed := true + THEN the distance should be NULL, confirming the parity +1 left the NULL + short-circuit intact in the combined variant + """ + # Arrange + sql = """ + SELECT DISTANCE(a.interval, b.interval, stranded := true, signed := true) + as distance + FROM (SELECT 'chr1' as chrom, 100 as start, 200 as end, '.' as strand) a + CROSS JOIN (SELECT 'chr1' as chrom, 200 as start, 300 as end, '+' as strand) b + """ + + # Act + result = _run(sql) + + # Assert + assert result is None, f"Expected NULL for '.' strand, got {result}" From be8e0815d64e21e2daac777d371b7e12e064411c Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 11 Jun 2026 22:46:32 -0400 Subject: [PATCH 063/142] docs: Document bedtools parity semantics for DISTANCE State that book-ended (adjacent) intervals return 1 and a non-overlapping gap returns the half-open gap plus one, matching bedtools closest -d, while overlapping intervals still return 0. --- docs/dialect/distance-operators.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/dialect/distance-operators.rst b/docs/dialect/distance-operators.rst index 5486ca9..d3ebe89 100644 --- a/docs/dialect/distance-operators.rst +++ b/docs/dialect/distance-operators.rst @@ -16,10 +16,11 @@ Description ~~~~~~~~~~~ The ``DISTANCE`` operator returns the number of base pairs separating two genomic -intervals. It follows standard genomic distance conventions: +intervals, matching ``bedtools closest -d`` semantics: - **Overlapping intervals**: Returns ``0`` -- **Non-overlapping intervals**: Returns the gap in base pairs (positive integer) +- **Book-ended (adjacent) intervals** (``A.end == B.start`` in half-open coordinates): Returns ``1`` +- **Non-overlapping intervals**: Returns the half-open gap plus one (a raw gap of ``N`` bases reports ``N + 1``) - **Different chromosomes**: Returns ``NULL`` Syntax @@ -42,7 +43,8 @@ Return Value ~~~~~~~~~~~~ - ``0`` for overlapping intervals -- Positive integer (gap in base pairs) for non-overlapping same-chromosome intervals +- ``1`` for book-ended (adjacent) intervals, matching ``bedtools closest -d`` +- Positive integer (half-open gap ``+ 1``) for non-overlapping same-chromosome intervals - ``NULL`` for intervals on different chromosomes Examples From 9f2ae39fa91740c9e391182a4526c368acce58b4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 12 Jun 2026 00:19:15 -0400 Subject: [PATCH 064/142] docs: Correct MCP DISTANCE operator return description The explain_operator metadata for DISTANCE still described the old half-open gap ("positive integer for gap"), which contradicted the operator docstring and distance-operators.rst after the bedtools parity fix. Describe the corrected semantics: 0 for overlapping, 1 for book-ended, half-open gap plus one otherwise, NULL across chromosomes. --- src/giql/mcp/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/giql/mcp/server.py b/src/giql/mcp/server.py index 08fb7a8..949bbf8 100644 --- a/src/giql/mcp/server.py +++ b/src/giql/mcp/server.py @@ -65,7 +65,7 @@ {"name": "interval_a", "description": "First genomic interval"}, {"name": "interval_b", "description": "Second genomic interval"}, ], - "returns": "0 for overlapping, positive integer for gap, NULL for different chromosomes", + "returns": "0 for overlapping, 1 for book-ended, half-open gap + 1 for a gap, NULL for different chromosomes", "example": "SELECT DISTANCE(a.interval, b.interval) AS dist FROM a CROSS JOIN b", "doc_file": "dialect/distance-operators.rst", }, From fd9ca8d26348b41870ea60b044ea7b1d85348b50 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sat, 13 Jun 2026 06:07:30 -0400 Subject: [PATCH 065/142] feat: Add target model and resolve dialect param to a target Promote the transpile() dialect parameter into a first-class target selector. Introduce a Capabilities descriptor and Generic, DuckDB, and DataFusion target classes, each carrying a capability set and the sqlglot output dialect, plus a resolve_target() helper. The dialect string now resolves to a Target, and the DuckDB IEJoin gate is keyed off the range_join_strategy capability rather than a literal dialect comparison. Adds "datafusion" as an accepted target that routes through the generic binned path. Behaviour-preserving: existing dialect values emit identical SQL. --- src/giql/targets.py | 157 ++++++++++++++++++++++++++++++++++++++++++ src/giql/transpile.py | 30 ++++---- 2 files changed, 174 insertions(+), 13 deletions(-) create mode 100644 src/giql/targets.py diff --git a/src/giql/targets.py b/src/giql/targets.py new file mode 100644 index 0000000..ec38bb8 --- /dev/null +++ b/src/giql/targets.py @@ -0,0 +1,157 @@ +"""Target-engine model for GIQL transpilation. + +A :class:`Target` is a first-class SQL target engine: it carries a +:class:`Capabilities` set describing what the engine supports and the +sqlglot output dialect used to serialize standard AST for that engine. + +This is step 1 of epic #137. The targets and capability descriptors defined +here are the foundation that later steps build on — the operator-expander +registry is keyed by ``(target, operator)`` and emission choices become +capability lookups rather than scattered ``if dialect == ...`` branches. +At this step the model is wired into :func:`giql.transpile.transpile` only +to resolve the ``dialect`` parameter and drive the existing DuckDB IEJoin +gate; no emission behaviour changes. +""" + +from dataclasses import dataclass +from typing import Literal + +RangeJoinStrategy = Literal["binned", "iejoin"] + + +@dataclass(frozen=True) +class Capabilities: + """Feature set of a SQL target engine. + + Each field is a portable choice that later steps of epic #137 turn into + a capability lookup instead of a hardcoded dialect branch. + + Parameters + ---------- + supports_lateral : bool + Whether the engine supports ``LATERAL`` / correlated joins. Drives + the NEAREST LATERAL-vs-window-function strategy (#142). + supports_star_replace : bool + Whether the engine supports ``SELECT * REPLACE (...)``. Drives the + coordinate-canonicalization output: ``* REPLACE`` where supported, + an explicit portable projection otherwise (#143). Supported by + DuckDB / BigQuery / Snowflake / ClickHouse; not by PostgreSQL, + SQLite, or DataFusion. + supports_qualify : bool + Whether the engine supports the ``QUALIFY`` clause. + range_join_strategy : RangeJoinStrategy + The plan used for column-to-column INTERSECTS joins: ``"binned"`` + for the generic binned equi-join, ``"iejoin"`` for DuckDB's + per-partition IEJoin plan. + """ + + supports_lateral: bool + supports_star_replace: bool + supports_qualify: bool + range_join_strategy: RangeJoinStrategy + + +class Target: + """A SQL target engine. + + Subclasses declare the engine ``name``, the ``sqlglot_dialect`` used to + serialize AST for that engine (``None`` selects sqlglot's default + generic serialization), and the engine ``capabilities``. + """ + + name: str + sqlglot_dialect: str | None + capabilities: Capabilities + + +class GenericTarget(Target): + """Portable SQL-92-ish target with no engine-specific features. + + This is the default target (``dialect=None``). Its capabilities are the + conservative, maximally portable baseline that matches today's + :class:`giql.generators.base.BaseGIQLGenerator` output. + """ + + name = "generic" + sqlglot_dialect = None + capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=False, + range_join_strategy="binned", + ) + + +class DuckDBTarget(Target): + """DuckDB target. + + Serializes through sqlglot's ``duckdb`` dialect and uses the IEJoin + per-partition plan for column-to-column INTERSECTS joins. + """ + + name = "duckdb" + sqlglot_dialect = "duckdb" + capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=True, + supports_qualify=True, + range_join_strategy="iejoin", + ) + + +class DataFusionTarget(Target): + """Apache DataFusion target. + + sqlglot has no DataFusion dialect, so serialization falls back to the + generic form (``sqlglot_dialect = None``) for now; #145 finalizes + DataFusion serialization. The capability values below are conservative + and provisional — they are validated against a real DataFusion engine + when the operator migrations exercise them (#142, #145). DataFusion + supports ``* EXCEPT`` / ``* EXCLUDE`` but not ``* REPLACE``. + """ + + name = "datafusion" + sqlglot_dialect = None + capabilities = Capabilities( + supports_lateral=False, + supports_star_replace=False, + supports_qualify=False, + range_join_strategy="binned", + ) + + +_TARGETS_BY_NAME: dict[str, type[Target]] = { + GenericTarget.name: GenericTarget, + DuckDBTarget.name: DuckDBTarget, + DataFusionTarget.name: DataFusionTarget, +} + + +def resolve_target(dialect: str | None) -> Target: + """Resolve a ``dialect`` parameter to a :class:`Target` instance. + + Parameters + ---------- + dialect : str | None + The target dialect name. ``None`` resolves to :class:`GenericTarget`; + ``"duckdb"`` and ``"datafusion"`` resolve to their respective targets. + + Returns + ------- + Target + The resolved target instance. + + Raises + ------ + ValueError + If *dialect* is not a recognized target name. + """ + if dialect is None: + return GenericTarget() + + target_cls = _TARGETS_BY_NAME.get(dialect) + if target_cls is None or target_cls is GenericTarget: + raise ValueError( + f"Unknown dialect: {dialect!r}. Supported: 'duckdb', 'datafusion', or None." + ) + return target_cls() diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 5bfe894..912c430 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -17,6 +17,7 @@ from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables +from giql.targets import resolve_target from giql.transformer import ClusterTransformer from giql.transformer import IntersectsBinnedJoinTransformer from giql.transformer import IntersectsDuckDBIEJoinTransformer @@ -28,7 +29,7 @@ def transpile( giql: str, tables: list[str | Table] | None = None, *, - dialect: None = None, + dialect: Literal["datafusion"] | None = None, intersects_bin_size: int | None = None, ) -> str: ... @@ -47,7 +48,7 @@ def transpile( giql: str, tables: list[str | Table] | None = None, *, - dialect: Literal["duckdb"] | None = None, + dialect: Literal["duckdb", "datafusion"] | None = None, intersects_bin_size: int | None = None, ) -> str: """Transpile a GIQL query to SQL. @@ -64,16 +65,19 @@ def transpile( Table configurations. Strings use default column mappings (chrom, start, end, strand). :class:`Table` objects provide custom column name mappings. - dialect : Literal["duckdb"] | None - Optional target dialect. When set to ``"duckdb"``, column-to-column + dialect : Literal["duckdb", "datafusion"] | None + Optional target engine. Resolves to a :class:`giql.targets.Target` + carrying the engine's capability set; ``None`` selects the generic + portable target. When set to ``"duckdb"``, column-to-column ``INTERSECTS`` joins (INNER, SEMI, or ANTI) are transpiled into a per-chromosome dynamic-SQL pattern (``SET VARIABLE`` + ``query(getvariable(...))``) that DuckDB plans through its - range-join family (``IE_JOIN`` / ``PIECEWISE_MERGE_JOIN``). - Mutually exclusive with ``intersects_bin_size``. Defaults to - ``None`` (the generic binned equi-join path). Hard-error projection - shapes raise ``ValueError`` at transpile time; see the performance - guide for the full enumeration. + range-join family (``IE_JOIN`` / ``PIECEWISE_MERGE_JOIN``); this + IEJoin plan is mutually exclusive with ``intersects_bin_size``. + ``"datafusion"`` and ``None`` use the generic binned equi-join path + and accept ``intersects_bin_size``. Hard-error projection shapes + raise ``ValueError`` at transpile time; see the performance guide + for the full enumeration. intersects_bin_size : int | None Bin size for INTERSECTS equi-join optimization. When a query contains a full-table column-to-column INTERSECTS join, the @@ -136,9 +140,9 @@ def transpile( dialect="duckdb", ) """ - if dialect is not None and dialect != "duckdb": - raise ValueError(f"Unknown dialect: {dialect!r}. Supported: 'duckdb' or None.") - if dialect == "duckdb" and intersects_bin_size is not None: + target = resolve_target(dialect) + uses_iejoin = target.capabilities.range_join_strategy == "iejoin" + if uses_iejoin and intersects_bin_size is not None: raise ValueError( "intersects_bin_size has no effect with dialect='duckdb'; " "the DuckDB dialect uses an IEJoin per-partition plan instead " @@ -153,7 +157,7 @@ def transpile( # Falls back to the binned plan for unsupported shapes — see # IntersectsDuckDBIEJoinTransformer.transform_to_sql for the complete # fallback set. - if dialect == "duckdb": + if uses_iejoin: duckdb_transformer = IntersectsDuckDBIEJoinTransformer(tables_container) with _reraise_as_value_error("Transformation error"): duckdb_sql = duckdb_transformer.transform_to_sql(ast) From 451eb154eeffa1cbf1e4d4741d95b0a9a0446dc3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sat, 13 Jun 2026 06:07:46 -0400 Subject: [PATCH 066/142] test: Cover the target model and DataFusion dialect routing Add unit tests for Capabilities, the three target classes, and resolve_target, including parametrized and property-based rejection of unsupported dialect names. Extend the transpile dialect tests with datafusion-equals-generic identity across the operator spread and the duckdb bin-size guard boundaries. Add a DataFusion integration smoke suite that executes dialect="datafusion" output against a real DataFusion engine, covering the literal-predicate and binned-join routing seams. --- tests/integration/datafusion/__init__.py | 0 tests/integration/datafusion/conftest.py | 49 +++ .../datafusion/test_datafusion_target.py | 123 ++++++++ tests/test_targets.py | 295 ++++++++++++++++++ tests/test_transpile.py | 192 ++++++++++++ 5 files changed, 659 insertions(+) create mode 100644 tests/integration/datafusion/__init__.py create mode 100644 tests/integration/datafusion/conftest.py create mode 100644 tests/integration/datafusion/test_datafusion_target.py create mode 100644 tests/test_targets.py diff --git a/tests/integration/datafusion/__init__.py b/tests/integration/datafusion/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/datafusion/conftest.py b/tests/integration/datafusion/conftest.py new file mode 100644 index 0000000..f57d7bc --- /dev/null +++ b/tests/integration/datafusion/conftest.py @@ -0,0 +1,49 @@ +"""Fixtures for DataFusion target execution smoke tests (issue #132). + +These tests prove that SQL produced via ``transpile(..., dialect="datafusion")`` +parses and executes on a real DataFusion engine. The broad cross-target result +oracle and the full DataFusion integration lane are deferred to #139. +""" + +import pytest + +pytest.importorskip("datafusion") +pytest.importorskip("pyarrow") + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def datafusion_ctx(): + """Return a builder that registers interval tables into a SessionContext. + + The returned callable accepts ``name=[(chrom, start, end), ...]`` keyword + arguments and yields a DataFusion ``SessionContext`` with each table + registered under the default ``chrom``/``start``/``end`` schema (matching + the default :class:`giql.Table` column mapping). + """ + import pyarrow as pa + from datafusion import SessionContext + + schema = pa.schema( + [ + ("chrom", pa.utf8()), + ("start", pa.int64()), + ("end", pa.int64()), + ] + ) + + def _build(**tables): + ctx = SessionContext() + for name, rows in tables.items(): + arrays = { + "chrom": [r[0] for r in rows], + "start": [r[1] for r in rows], + "end": [r[2] for r in rows], + } + ctx.register_record_batches( + name, [pa.table(arrays, schema=schema).to_batches()] + ) + return ctx + + return _build diff --git a/tests/integration/datafusion/test_datafusion_target.py b/tests/integration/datafusion/test_datafusion_target.py new file mode 100644 index 0000000..8c267f5 --- /dev/null +++ b/tests/integration/datafusion/test_datafusion_target.py @@ -0,0 +1,123 @@ +"""DataFusion execution smoke tests for the ``dialect="datafusion"`` target. + +Issue #132 makes ``"datafusion"`` a routing key in :func:`giql.transpile`. The +existing ``TestBinnedJoinDataFusion`` suite proves the binned plan executes on +DataFusion, but always with ``dialect`` omitted (``None``); nothing executes the +``dialect="datafusion"`` entry point end-to-end. These tests close that seam. +""" + +import pytest + +from giql import transpile + +pytestmark = pytest.mark.integration + + +class TestDataFusionTargetExecution: + """End-to-end execution of dialect="datafusion" output on DataFusion.""" + + def test_intersects_literal_returns_overlapping_row(self, datafusion_ctx): + """Test a literal INTERSECTS query through the datafusion target. + + Given: + A peaks table in a real DataFusion context with one overlapping + interval, one non-overlapping interval, and one on another + chromosome. + When: + A literal-range INTERSECTS query is transpiled with + dialect="datafusion" and executed. + Then: + It should parse, execute, and return exactly the overlapping + chr1 row. + """ + # Arrange + ctx = datafusion_ctx( + peaks=[ + ("chr1", 1500, 1800), # overlaps chr1:1000-2000 + ("chr1", 5000, 6000), # same chrom, no overlap + ("chr2", 1500, 1800), # overlapping position, wrong chrom + ] + ) + sql = transpile( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS 'chr1:1000-2000'", + tables=["peaks"], + dialect="datafusion", + ) + + # Act + df = ctx.sql(sql).to_pandas() + + # Assert + assert len(df) == 1 + assert df.iloc[0]["chrom"] == "chr1" + assert df.iloc[0]["start"] == 1500 + + def test_intersects_literal_no_overlap_returns_zero_rows(self, datafusion_ctx): + """Test that a non-overlapping literal INTERSECTS yields no rows. + + Given: + A peaks table in a real DataFusion context with no interval + overlapping the query range. + When: + The literal INTERSECTS query is transpiled with + dialect="datafusion" and executed. + Then: + It should execute and return zero rows. + """ + # Arrange + ctx = datafusion_ctx( + peaks=[ + ("chr1", 5000, 6000), + ("chr2", 1500, 1800), + ] + ) + sql = transpile( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS 'chr1:1000-2000'", + tables=["peaks"], + dialect="datafusion", + ) + + # Act + df = ctx.sql(sql).to_pandas() + + # Assert + assert len(df) == 0 + + def test_intersects_join_with_bin_size_dedups_across_bins(self, datafusion_ctx): + """Test the datafusion-routed binned join honours an explicit bin size. + + Given: + Two interval tables in a real DataFusion context whose single + intervals overlap and span several bins under a small bin size. + When: + A column-to-column INTERSECTS join is transpiled with + dialect="datafusion" and an explicit intersects_bin_size, then + executed. + Then: + It should be accepted (not rejected as it would be for duckdb), + route through the binned path, and return exactly one row — the + overlap, de-duplicated across the shared bins. + """ + # Arrange + ctx = datafusion_ctx( + peaks=[("chr1", 0, 5000)], + genes=[("chr1", 500, 4500)], + ) + sql = transpile( + 'SELECT a.chrom, a.start, a."end", ' + 'b.start AS b_start, b."end" AS b_end ' + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="datafusion", + intersects_bin_size=1000, + ) + + # Act + df = ctx.sql(sql).to_pandas() + + # Assert + assert len(df) == 1 + assert df.iloc[0]["start"] == 0 + assert df.iloc[0]["b_start"] == 500 diff --git a/tests/test_targets.py b/tests/test_targets.py new file mode 100644 index 0000000..1049ada --- /dev/null +++ b/tests/test_targets.py @@ -0,0 +1,295 @@ +"""Tests for the target-engine model.""" + +from dataclasses import FrozenInstanceError + +import pytest +from hypothesis import given +from hypothesis import strategies as st + +from giql.targets import Capabilities +from giql.targets import DataFusionTarget +from giql.targets import DuckDBTarget +from giql.targets import GenericTarget +from giql.targets import resolve_target + + +class TestCapabilities: + """Tests for the Capabilities dataclass.""" + + def test___init___stores_all_descriptors(self): + """Test that Capabilities records each descriptor. + + Given: + A full set of capability descriptor values. + When: + A Capabilities instance is constructed. + Then: + It should expose each value on the corresponding attribute. + """ + # Act + caps = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=True, + range_join_strategy="binned", + ) + + # Assert + assert caps.supports_lateral is True + assert caps.supports_star_replace is False + assert caps.supports_qualify is True + assert caps.range_join_strategy == "binned" + + def test___eq___with_identical_values(self): + """Test value equality of Capabilities. + + Given: + Two Capabilities built from identical descriptor values. + When: + They are compared for equality. + Then: + It should treat them as equal (frozen value semantics). + """ + # Arrange + first = Capabilities( + supports_lateral=True, + supports_star_replace=True, + supports_qualify=True, + range_join_strategy="iejoin", + ) + second = Capabilities( + supports_lateral=True, + supports_star_replace=True, + supports_qualify=True, + range_join_strategy="iejoin", + ) + + # Act & assert + assert first == second + + def test___eq___with_one_differing_field(self): + """Test value inequality of Capabilities. + + Given: + Two Capabilities identical in every descriptor except one. + When: + They are compared for equality. + Then: + It should treat them as unequal (per-field value semantics). + """ + # Arrange + base = Capabilities( + supports_lateral=True, + supports_star_replace=True, + supports_qualify=True, + range_join_strategy="iejoin", + ) + differing = Capabilities( + supports_lateral=True, + supports_star_replace=True, + supports_qualify=True, + range_join_strategy="binned", + ) + + # Act & assert + assert base != differing + + def test___setattr___is_frozen(self): + """Test that Capabilities is immutable. + + Given: + A constructed Capabilities instance. + When: + An attribute is reassigned. + Then: + It should raise FrozenInstanceError. + """ + # Arrange + caps = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=False, + range_join_strategy="binned", + ) + + # Act & assert + with pytest.raises(FrozenInstanceError): + caps.supports_lateral = False # type: ignore[misc] + + +class TestGenericTarget: + """Tests for the GenericTarget.""" + + def test___init___exposes_portable_baseline(self): + """Test the generic target's identity and capabilities. + + Given: + The GenericTarget class. + When: + An instance is constructed. + Then: + It should carry the portable SQL-92 baseline: no engine dialect, + lateral supported, no star-REPLACE, no QUALIFY, binned joins. + """ + # Act + target = GenericTarget() + + # Assert + assert target.name == "generic" + assert target.sqlglot_dialect is None + assert target.capabilities.supports_lateral is True + assert target.capabilities.supports_star_replace is False + assert target.capabilities.supports_qualify is False + assert target.capabilities.range_join_strategy == "binned" + + +class TestDuckDBTarget: + """Tests for the DuckDBTarget.""" + + def test___init___exposes_duckdb_features(self): + """Test the DuckDB target's identity and capabilities. + + Given: + The DuckDBTarget class. + When: + An instance is constructed. + Then: + It should serialize through the duckdb dialect and enable + lateral, star-REPLACE, QUALIFY, and the IEJoin strategy. + """ + # Act + target = DuckDBTarget() + + # Assert + assert target.name == "duckdb" + assert target.sqlglot_dialect == "duckdb" + assert target.capabilities.supports_lateral is True + assert target.capabilities.supports_star_replace is True + assert target.capabilities.supports_qualify is True + assert target.capabilities.range_join_strategy == "iejoin" + + +class TestDataFusionTarget: + """Tests for the DataFusionTarget.""" + + def test___init___exposes_conservative_capabilities(self): + """Test the DataFusion target's identity and capabilities. + + Given: + The DataFusionTarget class. + When: + An instance is constructed. + Then: + It should fall back to generic serialization (no sqlglot + DataFusion dialect), disable star-REPLACE and QUALIFY, and use + the binned join strategy. + """ + # Act + target = DataFusionTarget() + + # Assert + assert target.name == "datafusion" + assert target.sqlglot_dialect is None + assert target.capabilities.supports_lateral is False + assert target.capabilities.supports_star_replace is False + assert target.capabilities.supports_qualify is False + assert target.capabilities.range_join_strategy == "binned" + + +def test_resolve_target_with_none_returns_generic(): + """Test resolution of the default dialect. + + Given: + A None dialect argument. + When: + resolve_target is called. + Then: + It should return a GenericTarget. + """ + # Act + target = resolve_target(None) + + # Assert + assert isinstance(target, GenericTarget) + + +def test_resolve_target_with_duckdb_returns_duckdb(): + """Test resolution of the duckdb dialect. + + Given: + The dialect name "duckdb". + When: + resolve_target is called. + Then: + It should return a DuckDBTarget. + """ + # Act + target = resolve_target("duckdb") + + # Assert + assert isinstance(target, DuckDBTarget) + + +def test_resolve_target_with_datafusion_returns_datafusion(): + """Test resolution of the datafusion dialect. + + Given: + The dialect name "datafusion". + When: + resolve_target is called. + Then: + It should return a DataFusionTarget. + """ + # Act + target = resolve_target("datafusion") + + # Assert + assert isinstance(target, DataFusionTarget) + + +@pytest.mark.parametrize( + "dialect", + [ + "postgres", # plain unknown engine + "sqlite", # second unknown — guards against a hardcoded message + "DuckDB", # case-variant of a real target — lookup is case-sensitive + "", # empty string is distinct from None + "generic", # the internal target name — None is the only public path + ], +) +def test_resolve_target_with_unsupported_dialect_raises(dialect): + """Test resolution of dialect names that are not public targets. + + Given: + A dialect string that is not one of the supported public names + (an unknown engine, a case-variant, the empty string, or the + internal "generic" name). + When: + resolve_target is called. + Then: + It should raise ValueError naming the offending value and the + supported targets. + """ + # Act & assert + with pytest.raises( + ValueError, + match=rf"Unknown dialect: {dialect!r}\..*'duckdb', 'datafusion', or None\.", + ): + resolve_target(dialect) + + +@given(st.text().filter(lambda s: s not in ("duckdb", "datafusion"))) +def test_resolve_target_with_arbitrary_unsupported_string_raises(dialect): + """Test resolution over the open domain of unsupported strings. + + Given: + Any string that is not a registered public dialect name (the only + public string names are "duckdb" and "datafusion"). + When: + resolve_target is called. + Then: + It should raise ValueError rather than returning a target. + """ + # Act & assert + with pytest.raises(ValueError, match="Unknown dialect"): + resolve_target(dialect) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index e0f54fe..e43a1e0 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -389,6 +389,198 @@ def test_invalid_syntax(self): with pytest.raises(ValueError, match="Parse error"): transpile("SELECT * FORM peaks") # typo: FORM instead of FROM + def test_unknown_dialect(self): + """Test transpilation with an unrecognized dialect. + + Given: + A GIQL query and a dialect that is not a supported target. + When: + Transpiling. + Then: + It should raise ValueError naming the supported targets. + """ + # Act & assert + with pytest.raises(ValueError, match="Unknown dialect: 'postgres'"): + transpile( + "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'", + tables=["peaks"], + dialect="postgres", # type: ignore[call-overload] + ) + + +class TestTranspileDialects: + """Tests for dialect/target selection.""" + + def test_transpile_datafusion_accepted(self): + """Test that the datafusion dialect is a valid target. + + Given: + A GIQL query and dialect="datafusion". + When: + Transpiling. + Then: + It should return SQL referencing the table without raising. + """ + # Act + sql = transpile( + "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'", + tables=["peaks"], + dialect="datafusion", + ) + + # Assert + assert "SELECT" in sql + assert "peaks" in sql + + @pytest.mark.parametrize( + "query, tables", + [ + ( + "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1000-2000'", + ["peaks"], + ), + ("SELECT * FROM peaks WHERE interval CONTAINS 'chr1:1500'", ["peaks"]), + ( + "SELECT * FROM peaks WHERE interval WITHIN 'chr1:1000-2000'", + ["peaks"], + ), + ( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)", + ["genes"], + ), + ( + "SELECT a.chrom, b.chrom FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + ["peaks", "genes"], + ), + ( + "SELECT * FROM peaks WHERE interval INTERSECTS " + "ANY('chr1:1000-2000', 'chr1:5000-6000')", + ["peaks"], + ), + ], + ids=["intersects_literal", "contains", "within", "nearest", "join", "any"], + ) + def test_transpile_datafusion_matches_generic_output(self, query, tables): + """Test that datafusion routes through the generic path for now. + + Given: + A GIQL query across the operator spread, transpiled with + dialect=None and with dialect="datafusion". + When: + Comparing the two outputs. + Then: + It should produce byte-identical SQL, since datafusion has no + engine-specific expansion yet. + """ + # Act + generic_sql = transpile(query, tables=tables) + datafusion_sql = transpile(query, tables=tables, dialect="datafusion") + + # Assert + assert datafusion_sql == generic_sql + + def test_transpile_datafusion_accepts_intersects_bin_size(self): + """Test that datafusion honours the binned-join bin size identically. + + Given: + A column-to-column INTERSECTS join with an explicit + intersects_bin_size, transpiled with dialect=None and + dialect="datafusion". + When: + Comparing the two outputs. + Then: + It should produce byte-identical SQL — datafusion takes the same + binned-join path and honours the bin size. + """ + # Arrange + query = ( + "SELECT a.chrom, b.chrom FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + generic_sql = transpile( + query, tables=["peaks", "genes"], intersects_bin_size=50000 + ) + datafusion_sql = transpile( + query, + tables=["peaks", "genes"], + dialect="datafusion", + intersects_bin_size=50000, + ) + + # Assert + assert datafusion_sql == generic_sql + + def test_transpile_duckdb_returns_sql_for_column_join(self): + """Test the duckdb happy path for a column-to-column INTERSECTS join. + + Given: + A column-to-column INTERSECTS join and dialect="duckdb". + When: + Transpiling. + Then: + It should return SQL referencing the joined tables without raising. + """ + # Act + sql = transpile( + "SELECT a.chrom, a.start, b.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + ) + + # Assert + assert "SELECT" in sql + assert "peaks" in sql + assert "genes" in sql + + def test_transpile_duckdb_rejects_intersects_bin_size(self): + """Test that duckdb and intersects_bin_size are mutually exclusive. + + Given: + dialect="duckdb" combined with an explicit intersects_bin_size. + When: + Transpiling. + Then: + It should raise ValueError explaining the IEJoin plan ignores + the bin size. + """ + # Act & assert + with pytest.raises( + ValueError, + match=r"intersects_bin_size has no effect.*Pass one or the other, not both\.", + ): + transpile( + "SELECT a.chrom, b.chrom FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + intersects_bin_size=50000, + ) + + def test_transpile_duckdb_rejects_zero_intersects_bin_size(self): + """Test that a falsy-but-set bin size still triggers the duckdb guard. + + Given: + dialect="duckdb" combined with intersects_bin_size=0. + When: + Transpiling. + Then: + It should raise ValueError — the guard is `is not None`, so 0 + (falsy) still conflicts with the IEJoin plan. + """ + # Act & assert + with pytest.raises(ValueError, match="intersects_bin_size has no effect"): + transpile( + "SELECT a.chrom, b.chrom FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval", + tables=["peaks", "genes"], + dialect="duckdb", + intersects_bin_size=0, + ) + class TestModuleExports: """Tests for module-level exports.""" From 241d3104b7bb2f500bd2dbc69f2942953d9d1cbe Mon Sep 17 00:00:00 2001 From: Conrad Date: Sat, 13 Jun 2026 21:17:21 -0400 Subject: [PATCH 067/142] refactor: Make targets value-equal and harden dialect resolution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Convert Target and its subclasses to frozen dataclasses so resolved targets are value-equal and hashable — the contract the operator-expander registry will key on. Drop the generic entry from the name lookup so unknown dialects are rejected with a plain membership check instead of an identity special-case. Interpolate the target name into the bin-size mutual-exclusion error so it stays accurate as more IEJoin targets are added, and tighten the capability docstrings. --- src/giql/targets.py | 46 +++++++++++++++++++++++++++++-------------- src/giql/transpile.py | 4 ++-- 2 files changed, 33 insertions(+), 17 deletions(-) diff --git a/src/giql/targets.py b/src/giql/targets.py index ec38bb8..6fd29d3 100644 --- a/src/giql/targets.py +++ b/src/giql/targets.py @@ -29,8 +29,11 @@ class Capabilities: Parameters ---------- supports_lateral : bool - Whether the engine supports ``LATERAL`` / correlated joins. Drives - the NEAREST LATERAL-vs-window-function strategy (#142). + Whether the engine supports ``LATERAL`` / correlated joins. Will + drive the NEAREST LATERAL-vs-window-function strategy (#142). Until + then, :attr:`giql.generators.base.BaseGIQLGenerator.SUPPORTS_LATERAL` + remains the live source of truth at generation time; #142 reconciles + the two. supports_star_replace : bool Whether the engine supports ``SELECT * REPLACE (...)``. Drives the coordinate-canonicalization output: ``* REPLACE`` where supported, @@ -38,11 +41,14 @@ class Capabilities: DuckDB / BigQuery / Snowflake / ClickHouse; not by PostgreSQL, SQLite, or DataFusion. supports_qualify : bool - Whether the engine supports the ``QUALIFY`` clause. + Whether the engine supports the ``QUALIFY`` clause. Reserved: no + emission path consumes it yet (a future window-function operator + port would). range_join_strategy : RangeJoinStrategy The plan used for column-to-column INTERSECTS joins: ``"binned"`` for the generic binned equi-join, ``"iejoin"`` for DuckDB's - per-partition IEJoin plan. + per-partition IEJoin plan. The IEJoin path covers INNER / SEMI / + ANTI joins, with binned fallback for unsupported shapes. """ supports_lateral: bool @@ -51,12 +57,18 @@ class Capabilities: range_join_strategy: RangeJoinStrategy +@dataclass(frozen=True) class Target: """A SQL target engine. Subclasses declare the engine ``name``, the ``sqlglot_dialect`` used to serialize AST for that engine (``None`` selects sqlglot's default generic serialization), and the engine ``capabilities``. + + Targets are frozen, value-equal, and hashable: two ``DuckDBTarget()`` + instances compare equal and hash alike, so the operator-expander registry + (#138) can key on a resolved target by value. Equality is class-scoped — + ``GenericTarget() != DataFusionTarget()`` even where their fields overlap. """ name: str @@ -64,6 +76,7 @@ class Target: capabilities: Capabilities +@dataclass(frozen=True) class GenericTarget(Target): """Portable SQL-92-ish target with no engine-specific features. @@ -72,9 +85,9 @@ class GenericTarget(Target): :class:`giql.generators.base.BaseGIQLGenerator` output. """ - name = "generic" - sqlglot_dialect = None - capabilities = Capabilities( + name: str = "generic" + sqlglot_dialect: str | None = None + capabilities: Capabilities = Capabilities( supports_lateral=True, supports_star_replace=False, supports_qualify=False, @@ -82,6 +95,7 @@ class GenericTarget(Target): ) +@dataclass(frozen=True) class DuckDBTarget(Target): """DuckDB target. @@ -89,9 +103,9 @@ class DuckDBTarget(Target): per-partition plan for column-to-column INTERSECTS joins. """ - name = "duckdb" - sqlglot_dialect = "duckdb" - capabilities = Capabilities( + name: str = "duckdb" + sqlglot_dialect: str | None = "duckdb" + capabilities: Capabilities = Capabilities( supports_lateral=True, supports_star_replace=True, supports_qualify=True, @@ -99,6 +113,7 @@ class DuckDBTarget(Target): ) +@dataclass(frozen=True) class DataFusionTarget(Target): """Apache DataFusion target. @@ -110,9 +125,9 @@ class DataFusionTarget(Target): supports ``* EXCEPT`` / ``* EXCLUDE`` but not ``* REPLACE``. """ - name = "datafusion" - sqlglot_dialect = None - capabilities = Capabilities( + name: str = "datafusion" + sqlglot_dialect: str | None = None + capabilities: Capabilities = Capabilities( supports_lateral=False, supports_star_replace=False, supports_qualify=False, @@ -120,8 +135,9 @@ class DataFusionTarget(Target): ) +# Public dialect names only. ``generic`` is intentionally absent: ``None`` is +# the sole public way to select it (see :func:`resolve_target`). _TARGETS_BY_NAME: dict[str, type[Target]] = { - GenericTarget.name: GenericTarget, DuckDBTarget.name: DuckDBTarget, DataFusionTarget.name: DataFusionTarget, } @@ -150,7 +166,7 @@ def resolve_target(dialect: str | None) -> Target: return GenericTarget() target_cls = _TARGETS_BY_NAME.get(dialect) - if target_cls is None or target_cls is GenericTarget: + if target_cls is None: raise ValueError( f"Unknown dialect: {dialect!r}. Supported: 'duckdb', 'datafusion', or None." ) diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 912c430..f5462b4 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -144,8 +144,8 @@ def transpile( uses_iejoin = target.capabilities.range_join_strategy == "iejoin" if uses_iejoin and intersects_bin_size is not None: raise ValueError( - "intersects_bin_size has no effect with dialect='duckdb'; " - "the DuckDB dialect uses an IEJoin per-partition plan instead " + f"intersects_bin_size has no effect with dialect={target.name!r}; " + f"the {target.name} target uses an IEJoin per-partition plan instead " "of the binned equi-join. Pass one or the other, not both." ) From 0866e62bd0d72f7cff378157e66edf0b094aacbb Mon Sep 17 00:00:00 2001 From: Conrad Date: Sat, 13 Jun 2026 21:17:27 -0400 Subject: [PATCH 068/142] test: Pin target value semantics and harden datafusion bin-size check Add tests asserting targets are value-equal and hashable as registry keys. Harden the unsupported-dialect regex with re.escape and relabel the datafusion-equals-generic test as the alias regression net it is. Make the datafusion bin-size integration test assert the explicit bin size changes emission, so a dropped parameter would fail it. --- .../datafusion/test_datafusion_target.py | 18 ++++-- tests/test_targets.py | 62 +++++++++++++++++-- tests/test_transpile.py | 17 ++--- 3 files changed, 80 insertions(+), 17 deletions(-) diff --git a/tests/integration/datafusion/test_datafusion_target.py b/tests/integration/datafusion/test_datafusion_target.py index 8c267f5..96d32ea 100644 --- a/tests/integration/datafusion/test_datafusion_target.py +++ b/tests/integration/datafusion/test_datafusion_target.py @@ -97,27 +97,33 @@ def test_intersects_join_with_bin_size_dedups_across_bins(self, datafusion_ctx): executed. Then: It should be accepted (not rejected as it would be for duckdb), - route through the binned path, and return exactly one row — the - overlap, de-duplicated across the shared bins. + the explicit bin size should reach emission (changing the SQL vs + the default), and execution should return exactly one row — the + overlap, de-duplicated by the pairs CTE across the shared bins. """ # Arrange ctx = datafusion_ctx( peaks=[("chr1", 0, 5000)], genes=[("chr1", 500, 4500)], ) - sql = transpile( + query = ( 'SELECT a.chrom, a.start, a."end", ' 'b.start AS b_start, b."end" AS b_end ' - "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval" + ) + default_sql = transpile(query, tables=["peaks", "genes"], dialect="datafusion") + sized_sql = transpile( + query, tables=["peaks", "genes"], dialect="datafusion", intersects_bin_size=1000, ) # Act - df = ctx.sql(sql).to_pandas() + df = ctx.sql(sized_sql).to_pandas() # Assert - assert len(df) == 1 + assert sized_sql != default_sql # the explicit bin size reaches emission + assert len(df) == 1 # de-duplicated across the shared bins assert df.iloc[0]["start"] == 0 assert df.iloc[0]["b_start"] == 500 diff --git a/tests/test_targets.py b/tests/test_targets.py index 1049ada..14d10a6 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -1,5 +1,6 @@ """Tests for the target-engine model.""" +import re from dataclasses import FrozenInstanceError import pytest @@ -196,6 +197,57 @@ def test___init___exposes_conservative_capabilities(self): assert target.capabilities.range_join_strategy == "binned" +class TestTarget: + """Tests for the shared Target value semantics.""" + + def test___eq___same_target_type(self): + """Test that two instances of the same target are value-equal. + + Given: + Two separately constructed DuckDBTarget instances. + When: + They are compared for equality. + Then: + It should treat them as equal (frozen value semantics), so a + resolved target is a stable registry key. + """ + # Act & assert + assert DuckDBTarget() == DuckDBTarget() + + def test___eq___different_target_types(self): + """Test that distinct target classes are never equal. + + Given: + A GenericTarget and a DataFusionTarget (whose fields partly + overlap). + When: + They are compared for equality. + Then: + It should treat them as unequal, since equality is class-scoped. + """ + # Act & assert + assert GenericTarget() != DataFusionTarget() + + def test___hash___supports_set_and_dict_keys(self): + """Test that targets are hashable and usable as keys. + + Given: + Value-equal and value-distinct target instances. + When: + They are placed in a set and used as dict keys. + Then: + It should dedup equal instances and resolve a key built from a + separately constructed equal instance — the contract the + operator-expander registry (#138) relies on. + """ + # Arrange + registry = {(DuckDBTarget(), "Intersects"): "expander"} + + # Act & assert + assert len({DuckDBTarget(), DuckDBTarget(), GenericTarget()}) == 2 + assert registry[(DuckDBTarget(), "Intersects")] == "expander" + + def test_resolve_target_with_none_returns_generic(): """Test resolution of the default dialect. @@ -271,10 +323,12 @@ def test_resolve_target_with_unsupported_dialect_raises(dialect): supported targets. """ # Act & assert - with pytest.raises( - ValueError, - match=rf"Unknown dialect: {dialect!r}\..*'duckdb', 'datafusion', or None\.", - ): + pattern = ( + re.escape(f"Unknown dialect: {dialect!r}.") + + r".*" + + re.escape("'duckdb', 'datafusion', or None.") + ) + with pytest.raises(ValueError, match=pattern): resolve_target(dialect) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index e43a1e0..ea770a8 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -462,16 +462,19 @@ def test_transpile_datafusion_accepted(self): ids=["intersects_literal", "contains", "within", "nearest", "join", "any"], ) def test_transpile_datafusion_matches_generic_output(self, query, tables): - """Test that datafusion routes through the generic path for now. + """Test that datafusion is currently a pure alias for the generic target. Given: - A GIQL query across the operator spread, transpiled with - dialect=None and with dialect="datafusion". + A GIQL query transpiled with dialect=None and with + dialect="datafusion". The operator spread is a regression net, + not codepath coverage: datafusion has no engine-specific + expansion yet, so every query funnels through the same generic + path. When: Comparing the two outputs. Then: - It should produce byte-identical SQL, since datafusion has no - engine-specific expansion yet. + It should produce byte-identical SQL for every query, pinning + datafusion as a generic alias until later steps diverge it. """ # Act generic_sql = transpile(query, tables=tables) @@ -557,7 +560,7 @@ def test_transpile_duckdb_rejects_intersects_bin_size(self): "JOIN genes b ON a.interval INTERSECTS b.interval", tables=["peaks", "genes"], dialect="duckdb", - intersects_bin_size=50000, + intersects_bin_size=50000, # type: ignore[call-overload] ) def test_transpile_duckdb_rejects_zero_intersects_bin_size(self): @@ -578,7 +581,7 @@ def test_transpile_duckdb_rejects_zero_intersects_bin_size(self): "JOIN genes b ON a.interval INTERSECTS b.interval", tables=["peaks", "genes"], dialect="duckdb", - intersects_bin_size=0, + intersects_bin_size=0, # type: ignore[call-overload] ) From 7f3d2840d0427f7636ac54f209da84e71a98dedc Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 12:53:58 -0400 Subject: [PATCH 069/142] test: Add cross-target result oracle and DataFusion lane --- tests/integration/conftest.py | 254 ++++++++++++++++++ .../datafusion/test_cross_target_oracle.py | 213 +++++++++++++++ 2 files changed, 467 insertions(+) create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/datafusion/test_cross_target_oracle.py diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..afe4e6d --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,254 @@ +"""Cross-target result-oracle fixture for the operator-expansion epic (#137). + +The oracle is the verification backbone every later operator migration +(#140-#144) consumes: it transpiles one GIQL query for each registered target +(``Generic`` via ``dialect=None``, ``DuckDB`` via ``dialect="duckdb"``, +``DataFusion`` via ``dialect="datafusion"``), executes each target's SQL on the +engine that target is meant for, and asserts that every target's result set is +identical to the others and equal to an expected set. + +It compares ROWS, not SQL strings, on purpose: the DuckDB IEJoin SQL +(``SET VARIABLE`` / ``getvariable``) is DuckDB-only and will not even parse on +DataFusion. Result identity is the contract that survives that divergence. + +Engine routing per target: + +* ``"generic"`` (``dialect=None``) emits portable SQL -> executed on DataFusion. +* ``"datafusion"`` emits the DataFusion target SQL -> executed on DataFusion. +* ``"duckdb"`` emits the DuckDB-specific IEJoin SQL -> executed on DuckDB. + +Adding a new operator case is trivial: write one ``oracle(...)`` call with the +query, the table data, and the expected rows. Cases where a target cannot run +an operator yet (e.g. correlated ``LATERAL`` on DataFusion) restrict the run to +the supported targets via the ``targets`` argument. +""" + +from __future__ import annotations + +import pytest + +from giql import Table +from giql import transpile + +# Default per-target schema: GIQL's default interval columns. Reserved words +# (``end``) are quoted by the loaders below, mirroring ``test_binned_join.py``. +DEFAULT_COLUMNS: tuple[tuple[str, str], ...] = ( + ("chrom", "utf8"), + ("start", "int64"), + ("end", "int64"), +) + +# Target -> the engine the target's SQL is meant to execute on. +TARGET_ENGINE: dict[str, str] = { + "generic": "datafusion", + "datafusion": "datafusion", + "duckdb": "duckdb", +} + +# Default target spread: every target executed on its intended engine. +DEFAULT_TARGETS: tuple[str, ...] = ("generic", "datafusion", "duckdb") + +# transpile() dialect argument per target name. +_TARGET_DIALECT: dict[str, str | None] = { + "generic": None, + "datafusion": "datafusion", + "duckdb": "duckdb", +} + + +def _normalize(rows) -> list[tuple]: + """Return a row multiset as a sorted list of tuples for order-free compare. + + A list (not a set) preserves multiplicity so duplicate-row bugs surface; + sorting strips engine result ordering. Values are coerced to plain Python + scalars so DuckDB and DataFusion (pandas) rows compare equal. + """ + out = [] + for row in rows: + out.append(tuple(_scalar(v) for v in row)) + return sorted(out, key=lambda r: tuple((v is None, v) for v in r)) + + +def _scalar(value): + """Coerce engine-specific scalars to comparable plain Python values.""" + if value is None: + return None + # pandas / numpy nullable integers and NaN surface from DataFusion. + try: + import math + + if isinstance(value, float) and math.isnan(value): + return None + except (TypeError, ValueError): + pass + if hasattr(value, "item"): + try: + return value.item() + except (ValueError, AttributeError): + return value + return value + + +def _run_duckdb(sql: str, table_data: dict, columns) -> list[tuple]: + """Register tables in an in-memory DuckDB and return normalized result rows.""" + import duckdb + + conn = duckdb.connect(":memory:") + try: + for name, rows in table_data.items(): + cols_ddl = ", ".join( + f'"{col}" {"VARCHAR" if kind == "utf8" else "BIGINT"}' + for col, kind in columns + ) + conn.execute(f"CREATE TABLE {name} ({cols_ddl})") + if rows: + placeholders = ", ".join("?" for _ in columns) + conn.executemany( + f"INSERT INTO {name} VALUES ({placeholders})", + [tuple(r) for r in rows], + ) + return _normalize(conn.execute(sql).fetchall()) + finally: + conn.close() + + +def _run_datafusion(sql: str, table_data: dict, columns) -> list[tuple]: + """Register tables in a DataFusion context and return normalized result rows.""" + import pyarrow as pa + from datafusion import SessionContext + + pa_fields = [ + (col, pa.utf8() if kind == "utf8" else pa.int64()) for col, kind in columns + ] + schema = pa.schema(pa_fields) + + ctx = SessionContext() + for name, rows in table_data.items(): + arrays = { + col: [r[idx] for r in rows] for idx, (col, _kind) in enumerate(columns) + } + ctx.register_record_batches(name, [pa.table(arrays, schema=schema).to_batches()]) + return _normalize(ctx.sql(sql).to_pandas().itertuples(index=False, name=None)) + + +_ENGINE_RUNNERS = { + "duckdb": _run_duckdb, + "datafusion": _run_datafusion, +} + + +@pytest.fixture +def cross_target_oracle(): + """Return a callable asserting cross-target result identity for a GIQL query. + + The callable signature:: + + oracle( + query, + *, + expected, + tables=None, + columns=DEFAULT_COLUMNS, + targets=DEFAULT_TARGETS, + **table_data, + ) + + ``query`` + The GIQL query string. + ``expected`` + The expected result as an iterable of row tuples (order-free; compared + as a sorted multiset). + ``tables`` + Optional list of table names or :class:`giql.Table` specs passed to + :func:`giql.transpile`. Defaults to the keys of ``table_data`` using the + default column mapping. + ``columns`` + The ``(name, kind)`` schema (``kind`` in ``{"utf8", "int64"}``) used to + register ``table_data`` into both engines. Defaults to the GIQL default + ``chrom``/``start``/``end`` interval schema. + ``targets`` + The target names to exercise. Defaults to all three. Restrict this when + a target cannot run the operator on its engine yet (e.g. NEAREST's + correlated LATERAL has no DataFusion physical plan). + ``engines`` + Optional ``{target: engine}`` overrides for the default routing + (``generic``/``datafusion`` -> DataFusion, ``duckdb`` -> DuckDB). Use + this when a target's *portable* SQL must be executed on a different + engine for the case at hand — e.g. routing ``generic`` to DuckDB so a + LATERAL-based operator can still be compared against the duckdb target. + ``**table_data`` + ``name=[row, ...]`` table contents, where each row is a tuple matching + ``columns``. + + Returns the per-target normalized result so callers may make extra + assertions. + """ + + def _oracle( + query: str, + *, + expected, + tables=None, + columns=DEFAULT_COLUMNS, + targets=DEFAULT_TARGETS, + engines=None, + **table_data, + ) -> dict[str, list[tuple]]: + if tables is None: + tables = [_default_table(name, columns) for name in table_data] + + engine_for = dict(TARGET_ENGINE) + if engines: + engine_for.update(engines) + + expected_rows = _normalize(expected) + results: dict[str, list[tuple]] = {} + + for target in targets: + engine = engine_for[target] + importorskip = pytest.importorskip + importorskip("duckdb" if engine == "duckdb" else "datafusion") + if engine == "datafusion": + importorskip("pyarrow") + + sql = transpile(query, tables=tables, dialect=_TARGET_DIALECT[target]) + rows = _ENGINE_RUNNERS[engine](sql, table_data, columns) + results[target] = rows + + assert rows == expected_rows, ( + f"target {target!r} on engine {engine!r} returned {rows!r}, " + f"expected {expected_rows!r}\nSQL: {sql}" + ) + + # Cross-target identity: every executed target agrees row-for-row. + run_targets = list(results) + if len(run_targets) > 1: + reference = run_targets[0] + for other in run_targets[1:]: + assert results[other] == results[reference], ( + f"targets {reference!r} and {other!r} disagree: " + f"{results[reference]!r} != {results[other]!r}" + ) + + return results + + return _oracle + + +def _default_table(name, columns) -> Table: + """Build a :class:`giql.Table` mapping the canonical interval columns. + + The default chrom/start/end column names match :class:`giql.Table`'s own + defaults; any extra columns ride along as plain data columns the query can + still project. Custom column schemas should pass an explicit ``tables`` list + to the oracle instead. + """ + names = {col for col, _ in columns} + kwargs = {} + if "chrom" in names: + kwargs["chrom_col"] = "chrom" + if "start" in names: + kwargs["start_col"] = "start" + if "end" in names: + kwargs["end_col"] = "end" + return Table(name, **kwargs) diff --git a/tests/integration/datafusion/test_cross_target_oracle.py b/tests/integration/datafusion/test_cross_target_oracle.py new file mode 100644 index 0000000..91f17be --- /dev/null +++ b/tests/integration/datafusion/test_cross_target_oracle.py @@ -0,0 +1,213 @@ +"""Cross-target result-identity tests via the ``cross_target_oracle`` fixture. + +These exercise the reusable oracle (``tests/integration/conftest.py``) over the +operators that already emit identical generic SQL across Generic and DataFusion +and run correctly on DuckDB: INTERSECTS (literal + column-to-column join), +CONTAINS, WITHIN, and standalone NEAREST. No operator has been migrated to the +expander registry yet (epic #137), so this lane locks in the verification path +every later migration (#140-#144) will consume. + +NEAREST's expansion uses a correlated ``LATERAL`` subquery, which DataFusion has +no physical plan for today; its case therefore restricts the oracle to the +DuckDB-executed targets and is the one documented cross-target gap. +""" + +import pytest + +pytest.importorskip("duckdb") +pytest.importorskip("datafusion") +pytest.importorskip("pyarrow") + +pytestmark = pytest.mark.integration + + +class TestCrossTargetOracleIntersects: + """INTERSECTS identity across Generic, DataFusion, and DuckDB.""" + + def test_literal_intersects_returns_overlapping_rows(self, cross_target_oracle): + """Test a literal INTERSECTS yields identical rows on every target. + + Given: + A peaks table with one interval overlapping chr1:1000-2000, one + non-overlapping interval on chr1, and one on chr2. + When: + A literal-range INTERSECTS query runs through every target on its + engine. + Then: + Each target should return exactly the overlapping chr1 row and all + targets should agree. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS 'chr1:1000-2000'", + peaks=[ + ("chr1", 1500, 1800), + ("chr1", 5000, 6000), + ("chr2", 1500, 1800), + ], + expected=[("chr1", 1500, 1800)], + ) + + def test_literal_intersects_no_overlap_returns_zero_rows(self, cross_target_oracle): + """Test a non-overlapping literal INTERSECTS yields no rows on any target. + + Given: + A peaks table whose intervals do not overlap the query range. + When: + The literal INTERSECTS query runs through every target. + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS 'chr1:1000-2000'", + peaks=[ + ("chr1", 5000, 6000), + ("chr2", 1500, 1800), + ], + expected=[], + ) + + def test_column_to_column_intersects_join_agrees(self, cross_target_oracle): + """Test a column-to-column INTERSECTS join agrees across all targets. + + Given: + Peaks and genes tables where one peak overlaps one gene on chr1 and + other intervals do not overlap or sit on another chromosome. + When: + A column-to-column INTERSECTS join runs — generic/datafusion via the + binned equi-join on DataFusion and duckdb via the IEJoin on DuckDB. + Then: + Every target should return the single overlapping pair, proving the + structurally different DuckDB IEJoin SQL yields identical rows. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.chrom, a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + peaks=[ + ("chr1", 100, 500), + ("chr1", 1000, 2000), + ("chr2", 100, 500), + ], + genes=[ + ("chr1", 300, 600), + ("chr1", 5000, 6000), + ("chr2", 9000, 9500), + ], + expected=[("chr1", 100, 300)], + ) + + +class TestCrossTargetOracleContainsWithin: + """CONTAINS and WITHIN identity across all targets.""" + + def test_contains_returns_enclosing_rows(self, cross_target_oracle): + """Test CONTAINS yields identical enclosing rows across targets. + + Given: + A peaks table with one interval enclosing chr1:1200-1300, one that + only partially overlaps it, and one on chr2. + When: + A literal CONTAINS query runs through every target. + Then: + Every target should return only the fully-enclosing chr1 row. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval CONTAINS 'chr1:1200-1300'", + peaks=[ + ("chr1", 1000, 2000), + ("chr1", 1250, 1400), + ("chr2", 1000, 2000), + ], + expected=[("chr1", 1000, 2000)], + ) + + def test_within_returns_enclosed_rows(self, cross_target_oracle): + """Test WITHIN yields identical enclosed rows across targets. + + Given: + A peaks table with one interval fully inside chr1:1000-2000, one that + spills past it, and one on chr2. + When: + A literal WITHIN query runs through every target. + Then: + Every target should return only the fully-enclosed chr1 row. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval WITHIN 'chr1:1000-2000'", + peaks=[ + ("chr1", 1200, 1800), + ("chr1", 1500, 2500), + ("chr2", 1200, 1800), + ], + expected=[("chr1", 1200, 1800)], + ) + + +class TestCrossTargetOracleNearest: + """Standalone NEAREST identity on the targets whose engine supports LATERAL.""" + + def test_standalone_nearest_k1_agrees_on_duckdb_targets(self, cross_target_oracle): + """Test standalone NEAREST k=1 agrees across the LATERAL-capable targets. + + Given: + A single-row peaks table and three candidate genes at varying + distances on chr1. + When: + A correlated ``CROSS JOIN LATERAL NEAREST(..., k := 1)`` query runs + on the generic and duckdb targets, both executed on DuckDB. + Then: + Both targets should return the single nearest gene and agree. + + The generic target is routed to DuckDB here (not its default DataFusion + engine): the correlated LATERAL this expansion emits has no DataFusion + physical plan — the one documented cross-target gap. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.chrom, a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + peaks=[("chr1", 200, 300)], + genes=[ + ("chr1", 1000, 1100), + ("chr1", 50, 60), + ("chr1", 280, 290), + ], + expected=[("chr1", 200, 280)], + targets=("generic", "duckdb"), + engines={"generic": "duckdb"}, + ) + + def test_nearest_on_datafusion_lateral_is_unsupported(self, cross_target_oracle): + """Test the NEAREST LATERAL expansion has no DataFusion physical plan. + + Given: + The same single-row NEAREST query. + When: + The oracle is asked to run only the datafusion target on DataFusion. + Then: + DataFusion should reject the correlated LATERAL, pinning the gap that + justifies excluding it from the identity assertion above. + """ + # Arrange / Act + with pytest.raises(Exception) as excinfo: + cross_target_oracle( + "SELECT a.chrom, a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + peaks=[("chr1", 200, 300)], + genes=[("chr1", 280, 290)], + expected=[("chr1", 200, 280)], + targets=("datafusion",), + ) + + # Assert + assert "not implemented" in str(excinfo.value).lower() From 09230c8f0c157a75daaeabb7f374d780bd9c2eee Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 12:55:04 -0400 Subject: [PATCH 070/142] feat: Add operator-expander registry and ExpandOperators pass --- src/giql/expander.py | 398 ++++++++++++++++++++++++++++++++++++++++ src/giql/expressions.py | 19 ++ src/giql/transpile.py | 10 + 3 files changed, 427 insertions(+) create mode 100644 src/giql/expander.py diff --git a/src/giql/expander.py b/src/giql/expander.py new file mode 100644 index 0000000..35085ca --- /dev/null +++ b/src/giql/expander.py @@ -0,0 +1,398 @@ +"""The operator-expander registry and the ``ExpandOperators`` pass (epic #137). + +This module is step 2 of epic #137. It lands the dispatch infrastructure that the +remaining steps migrate each operator onto, one at a time: + +* an :class:`OperatorExpander` protocol — ``expand(node, ctx) -> exp.Expression``, + the unit that turns one GIQL operator node into standard sqlglot AST for the + active target; +* an :class:`ExpansionContext` carrying everything an expander needs — the pass-1 + :class:`~giql.resolver.OperatorResolution` metadata, the active + :class:`~giql.targets.Target` and its :class:`~giql.targets.Capabilities`, + collision-safe alias minting, and the registered :class:`~giql.table.Tables`; +* a :class:`ExpanderRegistry` keyed by ``(target, operator_type)`` resolving an + expander through the fallback chain ``(target, op)`` → ``(generic, op)`` → + the legacy ``*_sql`` emitter; +* a :func:`register` decorator — the **public extension hook** by which a user + adds a target or overrides an operator for one, e.g. + ``@register(DuckDBTarget, GIQLDisjoin)``; +* the :class:`ExpandOperators` pass, which runs after + :func:`giql.canonicalizer.canonicalize_coordinates`, walks the AST, and + replaces every opted-in GIQL operator node with the registry's expansion. + +Gating (mirrors ``GIQL_CANONICALIZE``) +-------------------------------------- +The pass is gated per operator by a ``GIQL_EXPAND`` class attribute on the +operator's expression class, exactly as pass 2 is gated by ``GIQL_CANONICALIZE``. +An operator takes the new AST-expansion path **only when both** hold: + +* its class sets ``GIQL_EXPAND = True``, and +* the registry resolves an expander for ``(active target, operator type)`` — + i.e. a ``(target, op)`` or ``(generic, op)`` expander is registered. + +Otherwise it falls through to the legacy ``*_sql`` emitter on +:class:`giql.generators.base.BaseGIQLGenerator`. As of this issue **no operator +sets ``GIQL_EXPAND`` and the registry is empty, so the pass is a strict no-op**: +no node is touched and the emitted SQL is byte-identical. Each later migration PR +(epic #137 steps 4-9) registers a generic expander, flips one operator's +``GIQL_EXPAND`` flag, and deletes that operator's ``*_sql`` method. +""" + +from __future__ import annotations + +from typing import Callable +from typing import Protocol +from typing import runtime_checkable + +from sqlglot import exp +from sqlglot.helper import name_sequence + +from giql.resolver import META_KEY +from giql.resolver import OperatorResolution +from giql.table import Tables +from giql.targets import GenericTarget +from giql.targets import Target + +__all__ = [ + "EXPAND_ALIAS_PREFIX", + "ExpansionContext", + "OperatorExpander", + "ExpanderRegistry", + "REGISTRY", + "register", + "expand_operators", + "ExpandOperators", +] + +#: The prefix every alias minted by :meth:`ExpansionContext.alias` carries. The +#: leading double underscore keeps synthesized names clear of user identifiers, +#: mirroring the ``__giql_canon_`` / ``__giql_dj_`` reserved prefixes. +EXPAND_ALIAS_PREFIX = "__giql_x_" + +#: The GIQL operator expression classes the pass inspects. Membership alone does +#: not opt an operator in: the per-class ``GIQL_EXPAND`` flag plus a registered +#: expander do (see module docstring). Imported lazily inside the pass to avoid a +#: module-level import cycle with :mod:`giql.expressions`. + + +class ExpansionContext: + """Everything an :class:`OperatorExpander` needs to expand one operator. + + A fresh context is built per operator node by :class:`ExpandOperators`. It + bundles the node's pass-1 resolution metadata together with the active + target, collision-safe alias minting, and the registered tables, so an + expander never has to reach back into the pass or re-resolve anything. + + Attributes + ---------- + node : exp.Expression + The GIQL operator node being expanded. + resolution : OperatorResolution | None + The metadata :func:`giql.resolver.resolve_operator_refs` attached to + *node* (its resolved reference slots, column operands, deferrals, and + de-canonicalization output tables). ``None`` only if the node was never + annotated — an internal invariant violation an expander may assert on. + target : Target + The active :class:`~giql.targets.Target`, exposing its + ``capabilities`` and ``sqlglot_dialect``. + tables : Tables + The registered :class:`~giql.table.Tables` container. + """ + + __slots__ = ("node", "resolution", "target", "tables", "_alias_seq") + + def __init__( + self, + node: exp.Expression, + resolution: OperatorResolution | None, + target: Target, + tables: Tables, + alias_seq: Callable[[], str] | None = None, + ) -> None: + self.node = node + self.resolution = resolution + self.target = target + self.tables = tables + # A single sequence is threaded across every context built for one + # ``ExpandOperators`` run so aliases minted for sibling operators never + # collide; a standalone context falls back to its own sequence. + self._alias_seq = alias_seq or name_sequence(EXPAND_ALIAS_PREFIX) + + @property + def capabilities(self): + """The active target's :class:`~giql.targets.Capabilities`.""" + return self.target.capabilities + + def alias(self) -> str: + """Mint a fresh, query-unique alias with the reserved expander prefix. + + Draws sequential names from a per-run :func:`sqlglot.helper.name_sequence` + so two expanders (or two slots of one expander) never collide. + """ + return self._alias_seq() + + +@runtime_checkable +class OperatorExpander(Protocol): + """Turns one GIQL operator node into standard sqlglot AST for a target. + + An expander is any callable-bearing object whose :meth:`expand` takes the + operator node and its :class:`ExpansionContext` and returns the sqlglot AST + the node expands to for the active target. The returned expression replaces + the operator node in the tree; the stock per-target serializer renders it. + + The protocol is :func:`~typing.runtime_checkable`, so the registry can assert + a registered object satisfies it. A plain function is *not* an + ``OperatorExpander`` (it has no ``expand`` method); register one by wrapping + it (see :func:`register`, which accepts either form). + """ + + def expand(self, node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: ... + + +#: A bare expander function — the form most expanders are written in. The +#: registry stores either an :class:`OperatorExpander` object or one of these. +ExpanderFn = Callable[[exp.Expression, ExpansionContext], exp.Expression] + + +def _as_callable(expander: OperatorExpander | ExpanderFn) -> ExpanderFn: + """Normalize an expander to a plain ``(node, ctx) -> Expression`` callable.""" + if isinstance(expander, OperatorExpander): + return expander.expand + if callable(expander): + return expander + raise TypeError( + "An operator expander must be an OperatorExpander (an object with an " + f"'expand' method) or a callable; got {type(expander).__name__}." + ) + + +class ExpanderRegistry: + """Maps ``(target, operator_type)`` to the expander that handles it. + + Resolution follows the epic's fallback chain so a generic expander written + once covers every target, and a per-target entry overrides it only where the + engine genuinely differs: + + 1. ``(target, op)`` — an expander registered for the *exact* active target; + 2. ``(generic, op)`` — the portable expander registered for + :class:`~giql.targets.GenericTarget`; + 3. ``None`` — no expander, so the caller keeps the legacy ``*_sql`` emitter. + + The registry keys on the frozen, value-equal :class:`~giql.targets.Target` + *instance* (two ``DuckDBTarget()`` compare and hash alike), so registering + against ``DuckDBTarget`` and resolving against a freshly built + ``DuckDBTarget()`` hit the same entry. Registering against the *class* + ``DuckDBTarget`` is a convenience the decorator instantiates. + """ + + def __init__(self) -> None: + self._expanders: dict[tuple[Target, type], ExpanderFn] = {} + + def register( + self, + target: Target, + operator: type, + expander: OperatorExpander | ExpanderFn, + ) -> None: + """Register *expander* for ``(target, operator)``, overriding any prior. + + Parameters + ---------- + target : Target + The target instance the expander handles. Use + :class:`~giql.targets.GenericTarget` for the portable fallback. + operator : type + The GIQL operator expression class (e.g. + :class:`~giql.expressions.GIQLDisjoin`). + expander : OperatorExpander | ExpanderFn + The expander object or function. A later registration for the same + key replaces an earlier one (last-write-wins override). + """ + self._expanders[(target, operator)] = _as_callable(expander) + + def resolve(self, target: Target, operator: type) -> ExpanderFn | None: + """Return the expander for ``(target, operator)`` via the fallback chain. + + Tries the exact ``(target, op)`` entry, then the + ``(GenericTarget(), op)`` fallback, then ``None`` (legacy emitter). + """ + fn = self._expanders.get((target, operator)) + if fn is not None: + return fn + if target != GenericTarget(): + fn = self._expanders.get((GenericTarget(), operator)) + if fn is not None: + return fn + return None + + def unregister(self, target: Target, operator: type) -> None: + """Drop the ``(target, operator)`` entry if present (test affordance).""" + self._expanders.pop((target, operator), None) + + def clear(self) -> None: + """Drop every registration (test affordance).""" + self._expanders.clear() + + def __contains__(self, key: tuple[Target, type]) -> bool: + return key in self._expanders + + +#: The process-wide registry the :func:`register` decorator writes to and the +#: :class:`ExpandOperators` pass reads from. Empty as of this issue, so the pass +#: is a strict no-op. +REGISTRY = ExpanderRegistry() + + +def register( + target: type[Target] | Target, operator: type +) -> Callable[[OperatorExpander | ExpanderFn], OperatorExpander | ExpanderFn]: + """Register an operator expander for ``(target, operator)`` — the extension hook. + + The public decorator by which a built-in or user-supplied expander is added + to the process-wide :data:`REGISTRY`:: + + @register(DuckDBTarget, GIQLDisjoin) + def expand_disjoin(node, ctx): ... + + Parameters + ---------- + target : type[Target] | Target + The target the expander handles, given as the target *class* (the usual + form — it is instantiated with its default capabilities) or an explicit + instance. Pass :class:`~giql.targets.GenericTarget` for the portable + expander every target falls back to. + operator : type + The GIQL operator expression class the expander handles. + + Returns + ------- + Callable + A decorator that registers its argument and returns it unchanged, so the + decorated object stays usable on its own. + """ + target_instance = target() if isinstance(target, type) else target + + def decorator( + expander: OperatorExpander | ExpanderFn, + ) -> OperatorExpander | ExpanderFn: + REGISTRY.register(target_instance, operator, expander) + return expander + + return decorator + + +def _giql_operators() -> tuple[type, ...]: + """Return the GIQL operator classes, imported lazily to avoid a cycle.""" + from giql.expressions import Contains + from giql.expressions import GIQLCluster + from giql.expressions import GIQLDisjoin + from giql.expressions import GIQLDistance + from giql.expressions import GIQLMerge + from giql.expressions import GIQLNearest + from giql.expressions import Intersects + from giql.expressions import SpatialSetPredicate + from giql.expressions import Within + + return ( + GIQLDisjoin, + GIQLNearest, + GIQLDistance, + GIQLCluster, + GIQLMerge, + Intersects, + Contains, + Within, + SpatialSetPredicate, + ) + + +def expand_operators( + expression: exp.Expression, + target: Target, + tables: Tables, + registry: ExpanderRegistry | None = None, +) -> exp.Expression: + """Replace each opted-in GIQL operator with its registry expansion. + + Pass 3 of the normalization pipeline (epic #137). Runs after + :func:`giql.canonicalizer.canonicalize_coordinates`. For every GIQL operator + node it dispatches to the new AST-expansion path **only when** the operator's + class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for + ``(target, operator type)`` through its fallback chain; otherwise the node is + left untouched and the legacy ``*_sql`` emitter handles it. + + The pass mutates and returns *expression* in place. **With no operator + flagged and an empty registry it is a strict no-op** and the emitted SQL is + byte-identical, so the existing suite is the migration oracle. + + Parameters + ---------- + expression : exp.Expression + The pass-1/pass-2-annotated AST. + target : Target + The active target whose expanders to dispatch to. + tables : Tables + The registered table configurations, threaded into each context. + registry : ExpanderRegistry | None + The registry to resolve against; defaults to the process-wide + :data:`REGISTRY`. + + Returns + ------- + exp.Expression + The same *expression*, with opted-in operator nodes replaced by their + target-specific expansions (none, while every flag is off / the registry + is empty). + """ + reg = registry if registry is not None else REGISTRY + operators = _giql_operators() + alias_seq = name_sequence(EXPAND_ALIAS_PREFIX) + + # Collect first, then mutate: replacing nodes mid-walk is unsafe. + pending: list[tuple[exp.Expression, ExpanderFn]] = [] + for node in expression.walk(): + if not isinstance(node, operators): + continue + if not getattr(node, "GIQL_EXPAND", False): + continue + fn = reg.resolve(target, type(node)) + if fn is None: + continue + pending.append((node, fn)) + + for node, fn in pending: + resolution = node.meta.get(META_KEY) + if not isinstance(resolution, OperatorResolution): + resolution = None + ctx = ExpansionContext(node, resolution, target, tables, alias_seq) + replacement = fn(node, ctx) + if replacement is not node: + node.replace(replacement) + + return expression + + +class ExpandOperators: + """Callable wrapper for :func:`expand_operators`, parallel to the transformers. + + The transpile pipeline composes transformer *objects* bound to their + :class:`~giql.table.Tables`; this wrapper gives the expansion pass the same + shape — construct it with the active target and tables, then call it on the + AST — so it slots into the pipeline uniformly after + :class:`giql.canonicalizer.canonicalize_coordinates`. + """ + + def __init__( + self, + target: Target, + tables: Tables, + registry: ExpanderRegistry | None = None, + ) -> None: + self.target = target + self.tables = tables + self.registry = registry + + def transform(self, expression: exp.Expression) -> exp.Expression: + """Run the expansion pass over *expression* and return it.""" + return expand_operators(expression, self.target, self.tables, self.registry) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index c6894d1..4ba384a 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -110,6 +110,14 @@ class SpatialPredicate(exp.Binary): #: half-open) operands are left untouched and the emitted SQL stays byte-identical. _CANONICALIZE = True +#: Default opt-out from the ``ExpandOperators`` pass (epic #137, step 2). The +#: per-operator ``GIQL_EXPAND`` flag mirrors ``GIQL_CANONICALIZE``: an operator +#: takes the new AST-expansion path only when it sets ``GIQL_EXPAND = True`` *and* +#: an expander is registered for it; otherwise the legacy ``*_sql`` emitter runs. +#: Every operator defaults to ``False`` here, so the pass is a strict no-op until a +#: later migration step (#140+) flips one operator's flag alongside its expander. +_EXPAND = False + class Intersects(SpatialPredicate): """INTERSECTS spatial predicate. @@ -118,6 +126,7 @@ class Intersects(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE + GIQL_EXPAND = _EXPAND GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -132,6 +141,7 @@ class Contains(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE + GIQL_EXPAND = _EXPAND GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -146,6 +156,7 @@ class Within(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE + GIQL_EXPAND = _EXPAND GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -169,6 +180,7 @@ class SpatialSetPredicate(exp.Expression): } GIQL_CANONICALIZE = _CANONICALIZE + GIQL_EXPAND = _EXPAND GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -220,6 +232,8 @@ class GIQLCluster(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } + GIQL_EXPAND = _EXPAND + @classmethod def from_arg_list(cls, args): kwargs, positional_args = _split_named_and_positional(args) @@ -261,6 +275,8 @@ class GIQLMerge(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } + GIQL_EXPAND = _EXPAND + @classmethod def from_arg_list(cls, args): kwargs, positional_args = _split_named_and_positional(args) @@ -297,6 +313,7 @@ class GIQLDistance(exp.Func): } GIQL_CANONICALIZE = _CANONICALIZE + GIQL_EXPAND = _EXPAND GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -344,6 +361,7 @@ class GIQLNearest(exp.Func): #: half-open) operands are left untouched and the emitted SQL stays #: byte-identical. GIQL_CANONICALIZE = _CANONICALIZE + GIQL_EXPAND = _EXPAND GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), @@ -393,6 +411,7 @@ class GIQLDisjoin(exp.Func): #: (0-based half-open) operands are left unwrapped and the emitted SQL stays #: byte-identical. GIQL_CANONICALIZE = True + GIQL_EXPAND = _EXPAND GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), diff --git a/src/giql/transpile.py b/src/giql/transpile.py index f5462b4..2b3ba6d 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -13,6 +13,7 @@ from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect +from giql.expander import ExpandOperators from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Table @@ -191,6 +192,15 @@ def transpile( with _reraise_as_value_error("Canonicalization error"): ast = canonicalize_coordinates(ast) + # Pass 3 of the normalization pipeline (epic #137): replace each opted-in + # GIQL operator node with the AST its registered expander produces for the + # active target. No operator sets GIQL_EXPAND and the registry is empty, so + # this is a strict no-op until the per-operator migration issues land; the + # legacy *_sql emitters on the generator remain the fallback. + expand_operators = ExpandOperators(target, tables_container) + with _reraise_as_value_error("Expansion error"): + ast = expand_operators.transform(ast) + with _reraise_as_value_error("Transpilation error"): sql = generator.generate(ast) From e1213e8caa5a9b42d9c86ad1458584822bfbd34b Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 12:56:30 -0400 Subject: [PATCH 071/142] test: Cover the expander registry and ExpandOperators pass --- tests/test_expander.py | 599 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 599 insertions(+) create mode 100644 tests/test_expander.py diff --git a/tests/test_expander.py b/tests/test_expander.py new file mode 100644 index 0000000..cc4667e --- /dev/null +++ b/tests/test_expander.py @@ -0,0 +1,599 @@ +"""Tests for the operator-expander registry and the ``ExpandOperators`` pass. + +These tests pin the dispatch infrastructure of epic #137, step 2: + +* the :class:`~giql.expander.ExpanderRegistry` resolves an expander through the + ``(target, op)`` → ``(generic, op)`` → ``None`` (legacy) fallback chain, and a + later registration overrides an earlier one; +* the :func:`~giql.expander.register` decorator writes to the process-wide + registry and returns the expander unchanged; +* the :class:`~giql.expander.ExpandOperators` pass dispatches an opted-in + operator to its registered expander — including a *fake* custom target plus a + fake operator, the public extension hook; +* with nothing flagged and an empty registry the pass is a strict no-op and the + emitted SQL is byte-identical. +""" + +from dataclasses import dataclass + +import pytest +from sqlglot import exp +from sqlglot import parse_one + +from giql.dialect import GIQLDialect +from giql.expander import EXPAND_ALIAS_PREFIX +from giql.expander import REGISTRY +from giql.expander import ExpanderRegistry +from giql.expander import ExpandOperators +from giql.expander import ExpansionContext +from giql.expander import OperatorExpander +from giql.expander import expand_operators +from giql.expander import register +from giql.expressions import GIQLDisjoin +from giql.resolver import resolve_operator_refs +from giql.table import Table +from giql.table import Tables +from giql.targets import Capabilities +from giql.targets import DataFusionTarget +from giql.targets import DuckDBTarget +from giql.targets import GenericTarget +from giql.targets import Target +from giql.transpile import transpile + + +def _record(label: str): + """Build an expander function that replaces a node with a tagged literal.""" + + def _expander(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: + return exp.Literal.string(label) + + return _expander + + +@pytest.fixture +def clean_registry(): + """Isolate the process-wide REGISTRY, restoring its contents afterward.""" + saved = dict(REGISTRY._expanders) + REGISTRY.clear() + yield REGISTRY + REGISTRY.clear() + REGISTRY._expanders.update(saved) + + +class TestExpanderRegistry: + """Tests for ExpanderRegistry registration and fallback resolution.""" + + def test_resolve_returns_registered_target_expander(self): + """Test that resolve returns the exact-target expander. + + Given: + A registry with an expander registered for (DuckDBTarget, op). + When: + Resolving (DuckDBTarget(), op). + Then: + It should return that exact-target expander. + """ + # Arrange + registry = ExpanderRegistry() + fn = _record("duckdb") + registry.register(DuckDBTarget(), GIQLDisjoin, fn) + + # Act + resolved = registry.resolve(DuckDBTarget(), GIQLDisjoin) + + # Assert + assert resolved is fn + + def test_resolve_falls_back_to_generic_expander(self): + """Test that resolve falls back from a target to the generic expander. + + Given: + A registry with an expander registered only for (GenericTarget, op). + When: + Resolving (DuckDBTarget(), op), for which no exact entry exists. + Then: + It should fall back to the generic expander. + """ + # Arrange + registry = ExpanderRegistry() + generic_fn = _record("generic") + registry.register(GenericTarget(), GIQLDisjoin, generic_fn) + + # Act + resolved = registry.resolve(DuckDBTarget(), GIQLDisjoin) + + # Assert + assert resolved is generic_fn + + def test_resolve_prefers_target_over_generic(self): + """Test that an exact-target entry shadows the generic fallback. + + Given: + A registry with both a (DuckDBTarget, op) and a (GenericTarget, op) + expander registered. + When: + Resolving (DuckDBTarget(), op). + Then: + It should return the exact-target expander, not the generic one. + """ + # Arrange + registry = ExpanderRegistry() + duckdb_fn = _record("duckdb") + generic_fn = _record("generic") + registry.register(DuckDBTarget(), GIQLDisjoin, duckdb_fn) + registry.register(GenericTarget(), GIQLDisjoin, generic_fn) + + # Act + resolved = registry.resolve(DuckDBTarget(), GIQLDisjoin) + + # Assert + assert resolved is duckdb_fn + + def test_resolve_returns_none_for_legacy_fallback(self): + """Test that resolve returns None when no expander is registered. + + Given: + An empty registry. + When: + Resolving any (target, op). + Then: + It should return None, signalling the legacy *_sql emitter fallback. + """ + # Arrange + registry = ExpanderRegistry() + + # Act + resolved = registry.resolve(DuckDBTarget(), GIQLDisjoin) + + # Assert + assert resolved is None + + def test_resolve_generic_target_does_not_self_fallback(self): + """Test that a generic-target resolve does not double-check generic. + + Given: + An empty registry. + When: + Resolving (GenericTarget(), op). + Then: + It should return None without raising (no redundant generic lookup). + """ + # Arrange + registry = ExpanderRegistry() + + # Act + resolved = registry.resolve(GenericTarget(), GIQLDisjoin) + + # Assert + assert resolved is None + + def test_register_overrides_existing_entry(self): + """Test that a later registration overrides an earlier one. + + Given: + A registry with an expander already registered for (target, op). + When: + Registering a second expander for the same key. + Then: + It should resolve to the second expander (last-write-wins). + """ + # Arrange + registry = ExpanderRegistry() + first = _record("first") + second = _record("second") + registry.register(DuckDBTarget(), GIQLDisjoin, first) + + # Act + registry.register(DuckDBTarget(), GIQLDisjoin, second) + + # Assert + assert registry.resolve(DuckDBTarget(), GIQLDisjoin) is second + + def test_register_keys_on_target_value_not_identity(self): + """Test that the registry keys on the frozen target value. + + Given: + An expander registered against one DuckDBTarget() instance. + When: + Resolving against a distinct, freshly built DuckDBTarget() instance. + Then: + It should hit the same entry (frozen value-equal target keys). + """ + # Arrange + registry = ExpanderRegistry() + fn = _record("duckdb") + registry.register(DuckDBTarget(), GIQLDisjoin, fn) + + # Act + resolved = registry.resolve(DuckDBTarget(), GIQLDisjoin) + + # Assert + assert DuckDBTarget() is not DuckDBTarget() + assert resolved is fn + + def test_register_accepts_expander_object(self): + """Test that an OperatorExpander object registers and resolves. + + Given: + An object with an expand method (an OperatorExpander). + When: + Registering it and resolving the key. + Then: + It should resolve to a callable that delegates to expand. + """ + + # Arrange + class _Expander: + def expand(self, node, ctx): + return exp.Literal.string("object") + + registry = ExpanderRegistry() + obj = _Expander() + assert isinstance(obj, OperatorExpander) + registry.register(GenericTarget(), GIQLDisjoin, obj) + + # Act + resolved = registry.resolve(GenericTarget(), GIQLDisjoin) + result = resolved(None, None) + + # Assert + assert result == exp.Literal.string("object") + + def test_register_rejects_non_callable(self): + """Test that registering a non-callable, non-expander raises. + + Given: + A value that is neither callable nor an OperatorExpander. + When: + Registering it. + Then: + It should raise TypeError. + """ + # Arrange + registry = ExpanderRegistry() + + # Act & assert + with pytest.raises(TypeError): + registry.register(GenericTarget(), GIQLDisjoin, object()) + + +class TestRegisterDecorator: + """Tests for the @register extension-hook decorator.""" + + def test_register_writes_to_process_registry(self, clean_registry): + """Test that the decorator registers in the process-wide registry. + + Given: + The decorator applied to an expander for (DuckDBTarget, op). + When: + Resolving that key against the process-wide registry. + Then: + It should resolve to the decorated expander. + """ + + # Arrange & act + @register(DuckDBTarget, GIQLDisjoin) + def _expander(node, ctx): + return exp.Literal.string("decorated") + + # Assert + assert clean_registry.resolve(DuckDBTarget(), GIQLDisjoin) is _expander + + def test_register_returns_expander_unchanged(self, clean_registry): + """Test that the decorator returns its target unchanged. + + Given: + An expander function. + When: + Decorating it with @register. + Then: + It should return the same function object (usable standalone). + """ + + # Arrange + def _expander(node, ctx): + return exp.Literal.string("x") + + # Act + decorated = register(GenericTarget, GIQLDisjoin)(_expander) + + # Assert + assert decorated is _expander + + def test_register_accepts_target_instance(self, clean_registry): + """Test that the decorator accepts a target instance, not just a class. + + Given: + The decorator applied with an explicit DuckDBTarget() instance. + When: + Resolving that key. + Then: + It should resolve to the decorated expander. + """ + + # Arrange & act + @register(DuckDBTarget(), GIQLDisjoin) + def _expander(node, ctx): + return exp.Literal.string("instance") + + # Assert + assert clean_registry.resolve(DuckDBTarget(), GIQLDisjoin) is _expander + + +class TestExpansionContext: + """Tests for the ExpansionContext value object.""" + + def test_capabilities_returns_target_capabilities(self): + """Test that capabilities exposes the active target's capability set. + + Given: + A context built with a DuckDBTarget. + When: + Reading its capabilities property. + Then: + It should be the target's Capabilities. + """ + # Arrange + target = DuckDBTarget() + ctx = ExpansionContext(exp.true(), None, target, Tables()) + + # Act + caps = ctx.capabilities + + # Assert + assert isinstance(caps, Capabilities) + assert caps is target.capabilities + + def test_alias_mints_unique_prefixed_names(self): + """Test that alias mints distinct names under the reserved prefix. + + Given: + A single ExpansionContext. + When: + Minting two aliases. + Then: + They should differ and both carry the expander prefix. + """ + # Arrange + ctx = ExpansionContext(exp.true(), None, GenericTarget(), Tables()) + + # Act + first = ctx.alias() + second = ctx.alias() + + # Assert + assert first != second + assert first.startswith(EXPAND_ALIAS_PREFIX) + assert second.startswith(EXPAND_ALIAS_PREFIX) + + +# A fake custom target standing in for a user-defined engine: the extension-hook +# seam must dispatch to it exactly as to the built-in targets. +@dataclass(frozen=True) +class _FakeTarget(Target): + name: str = "fake" + sqlglot_dialect: str | None = None + capabilities: Capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=False, + range_join_strategy="binned", + ) + + +class TestExpandOperatorsPass: + """Tests for the ExpandOperators dispatch pass.""" + + def test_transform_dispatches_to_registered_expander(self, clean_registry): + """Test that the pass replaces an opted-in node via its expander. + + Given: + A DISJOIN AST whose operator is flagged GIQL_EXPAND and an expander + registered for (GenericTarget, GIQLDisjoin). + When: + Running the ExpandOperators pass. + Then: + The DISJOIN node should be replaced by the expander's output. + """ + # Arrange + clean_registry.register(GenericTarget(), GIQLDisjoin, _record("expanded")) + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + result = pass_.transform(ast) + + # Assert + assert not list(result.find_all(GIQLDisjoin)) + assert result.find(exp.Literal).this == "expanded" + + def test_transform_dispatches_to_fake_custom_target(self, clean_registry): + """Test that the pass dispatches to a user-defined custom target. + + Given: + A fake user target and an expander registered for it via @register, + plus an opted-in DISJOIN operator. + When: + Running ExpandOperators with the fake target as active. + Then: + The pass should dispatch to the fake target's expander (the public + extension hook works end to end). + """ + + # Arrange + @register(_FakeTarget, GIQLDisjoin) + def _fake_expander(node, ctx): + assert ctx.target == _FakeTarget() + return exp.Literal.string("fake-target") + + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(_FakeTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + result = pass_.transform(ast) + + # Assert + assert result.find(exp.Literal).this == "fake-target" + + def test_transform_passes_resolution_and_target_in_context(self, clean_registry): + """Test that the expander receives the pass-1 metadata and active target. + + Given: + An opted-in DISJOIN with a registered expander that captures its ctx. + When: + Running the pass with a DataFusionTarget. + Then: + The context should carry the operator's resolution metadata, the + active target, and the registered tables. + """ + # Arrange + captured = {} + + def _expander(node, ctx): + captured["ctx"] = ctx + return exp.Literal.string("ok") + + clean_registry.register(DataFusionTarget(), GIQLDisjoin, _expander) + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(DataFusionTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + pass_.transform(ast) + + # Assert + ctx = captured["ctx"] + assert ctx.target == DataFusionTarget() + assert ctx.tables is tables + assert ctx.resolution is not None + assert ctx.resolution.operator == "GIQLDisjoin" + + def test_transform_skips_unflagged_operator(self, clean_registry): + """Test that an unflagged operator is left untouched even when registered. + + Given: + An expander registered for (GenericTarget, GIQLDisjoin) but the + operator's GIQL_EXPAND flag left at its default False. + When: + Running the pass. + Then: + The DISJOIN node should remain in the tree (gate requires both). + """ + # Arrange + clean_registry.register(GenericTarget(), GIQLDisjoin, _record("expanded")) + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act (GIQL_EXPAND is False by default — no opt-in context) + result = pass_.transform(ast) + + # Assert + assert list(result.find_all(GIQLDisjoin)) + + def test_transform_skips_flagged_operator_with_no_expander(self, clean_registry): + """Test that a flagged operator with no expander is left untouched. + + Given: + An opted-in DISJOIN but an empty registry (no expander resolves). + When: + Running the pass. + Then: + The DISJOIN node should remain (gate requires a registered expander). + """ + # Arrange + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + result = pass_.transform(ast) + + # Assert + assert list(result.find_all(GIQLDisjoin)) + + +class TestNoOpWhenInert: + """The pass touches nothing while no operator opts in and the registry is empty.""" + + def test_expand_operators_is_identity_when_registry_empty(self): + """Test that the standalone pass returns the AST unchanged when inert. + + Given: + A DISJOIN AST, no operator flagged, and an empty registry. + When: + Running expand_operators directly. + Then: + The same AST is returned with the DISJOIN node intact. + """ + # Arrange + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + + # Act + result = expand_operators(ast, GenericTarget(), tables, ExpanderRegistry()) + + # Assert + assert result is ast + assert list(result.find_all(GIQLDisjoin)) + + def test_transpile_sql_unchanged_with_pass_inert(self): + """Test that transpile output is byte-identical with the pass inert. + + Given: + A DISJOIN query, the default (empty) registry, and no operator flagged. + When: + Transpiling with the wired-in pass versus a pass-bypassed reference. + Then: + The SQL should match exactly and carry no expander alias prefix. + """ + # Arrange + query = "SELECT * FROM DISJOIN(variants)" + tables = _tables() + ast = _prepare(query, tables) + from giql.generators import BaseGIQLGenerator + + expected = BaseGIQLGenerator(tables=tables).generate(ast) + + # Act + actual = transpile(query, tables=[Table("variants")]) + + # Assert + assert actual == expected + assert EXPAND_ALIAS_PREFIX not in actual + + +def _tables(names=("variants",)) -> Tables: + """Build a Tables container registering each name with default encoding.""" + container = Tables() + for name in names: + container.register(name, Table(name)) + return container + + +def _prepare(query: str, tables: Tables) -> exp.Expression: + """Parse *query* and run pass 1 (resolution) over the resulting AST.""" + ast = parse_one(query, dialect=GIQLDialect) + return resolve_operator_refs(ast, tables) + + +class _opted_in: + """Context manager opting an operator class into GIQL_EXPAND for a test.""" + + def __init__(self, operator: type) -> None: + self._operator = operator + self._prior = operator.__dict__.get("GIQL_EXPAND", False) + + def __enter__(self): + self._operator.GIQL_EXPAND = True + return self._operator + + def __exit__(self, *exc): + self._operator.GIQL_EXPAND = self._prior + return False From 38d18df8fa7f2d3a12d177e1af2d3d3b95ef7210 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 19:45:59 -0400 Subject: [PATCH 072/142] docs: Document registry teardown seam and IEJoin skip --- src/giql/expander.py | 32 ++++++++++++++++++++++++++++++-- src/giql/transpile.py | 10 ++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/giql/expander.py b/src/giql/expander.py index 35085ca..34f742d 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -16,6 +16,14 @@ * a :func:`register` decorator — the **public extension hook** by which a user adds a target or overrides an operator for one, e.g. ``@register(DuckDBTarget, GIQLDisjoin)``; +* the registry's teardown seam — :meth:`ExpanderRegistry.unregister`, + :meth:`ExpanderRegistry.clear`, and ``in`` membership (``__contains__``) — the + **supported public API** for undoing a custom registration. A user who + registers a custom expander (e.g. in a test fixture or a plugin's teardown) + resets the process-wide :data:`REGISTRY` through these rather than reaching + into private state: ``REGISTRY.unregister(DuckDBTarget(), GIQLDisjoin)`` drops + one entry, ``REGISTRY.clear()`` drops them all, and + ``(DuckDBTarget(), GIQLDisjoin) in REGISTRY`` tests for one; * the :class:`ExpandOperators` pass, which runs after :func:`giql.canonicalizer.canonicalize_coordinates`, walks the AST, and replaces every opted-in GIQL operator node with the registry's expansion. @@ -227,14 +235,34 @@ def resolve(self, target: Target, operator: type) -> ExpanderFn | None: return None def unregister(self, target: Target, operator: type) -> None: - """Drop the ``(target, operator)`` entry if present (test affordance).""" + """Drop the ``(target, operator)`` entry if present. + + Part of the registry's **public teardown seam**: a caller that + registered a custom expander (a plugin, or a test fixture) undoes one + registration with this rather than mutating private state. Resolving the + same key afterward falls back through the chain to ``None``. Dropping an + absent key is a no-op. + """ self._expanders.pop((target, operator), None) def clear(self) -> None: - """Drop every registration (test affordance).""" + """Drop every registration. + + The bulk form of :meth:`unregister`, and the **public reset** for the + process-wide :data:`REGISTRY` — e.g. a test fixture that saves and + restores the registry around a body that registers custom expanders. + """ self._expanders.clear() def __contains__(self, key: tuple[Target, type]) -> bool: + """Whether an *exact* ``(target, operator)`` entry is registered. + + Tests the exact key only — it does **not** walk the generic fallback + chain :meth:`resolve` follows (``(GenericTarget(), op) in registry`` is + the only way ``(target, op)`` reports present via the generic entry). + Part of the public teardown seam, for asserting a registration landed or + was torn down. + """ return key in self._expanders diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 2b3ba6d..9ef2100 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -163,6 +163,16 @@ def transpile( with _reraise_as_value_error("Transformation error"): duckdb_sql = duckdb_transformer.transform_to_sql(ast) if duckdb_sql is not None: + # WARNING: this early return emits the legacy IEJoin SQL directly and + # SKIPS the normalization pipeline below — pass 1 (resolution), pass 2 + # (canonicalization), and pass 3 (ExpandOperators, constructed ~40 + # lines down). The ExpandOperators registry is therefore NOT consulted + # on this path: a flagged operator on an IEJoin-eligible duckdb query + # is left un-expanded. This is benign today (the registry is empty and + # no operator opts in), but any DuckDB-pathed operator migration (#141) + # must either run expansion BEFORE this early return or have the IEJoin + # transformer defer to the registry. See the strict-xfail + # characterization test pinning this gap in tests/test_expander.py. return duckdb_sql intersects_transformer = IntersectsBinnedJoinTransformer( From 5ce9d368576fcf52ea4510e6cadd8a4cdfa98f92 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 19:47:34 -0400 Subject: [PATCH 073/142] refactor: Extract cross-target oracle internals into _oracle --- tests/integration/_oracle.py | 245 +++++++++++++++++++++++ tests/integration/conftest.py | 159 ++++----------- tests/integration/datafusion/conftest.py | 28 ++- 3 files changed, 292 insertions(+), 140 deletions(-) create mode 100644 tests/integration/_oracle.py diff --git a/tests/integration/_oracle.py b/tests/integration/_oracle.py new file mode 100644 index 0000000..d6634e7 --- /dev/null +++ b/tests/integration/_oracle.py @@ -0,0 +1,245 @@ +"""Engine-free internals for the cross-target result oracle (epic #137, #139). + +This is an importable, *non-test* module: it holds the pure helpers the +``cross_target_oracle`` fixture (``tests/integration/conftest.py``) wraps, so +they can be unit-tested directly without spinning up DuckDB or DataFusion. + +Layout: + +* :func:`normalize` / :func:`scalar` -- coerce engine rows into a sorted, + order-free, type-normalized multiset for comparison. +* :func:`resolve_routing` -- the *pure* routing decision (target -> engine + + transpile dialect), pulled out of the run loop so it is unit-testable and so + unknown targets / engines raise a clear error rather than a bare ``KeyError`` + (Finding 7). +* :func:`assert_cross_target` -- the comparison core (Finding 1): assert the + engines agree *with each other* first, then assert the agreed result equals + ``expected``. Both assertions are genuinely reachable. +* :func:`register_record_batches` -- the single pyarrow loader dance reused by + :func:`run_datafusion` and the #132 ``datafusion_ctx`` fixture (Finding 8). +* :func:`run_duckdb` / :func:`run_datafusion` -- the per-engine runners. +""" + +from __future__ import annotations + +import math + +# Target -> the engine the target's SQL is meant to execute on. +TARGET_ENGINE: dict[str, str] = { + "generic": "datafusion", + "datafusion": "datafusion", + "duckdb": "duckdb", +} + +# transpile() dialect argument per target name. +TARGET_DIALECT: dict[str, str | None] = { + "generic": None, + "datafusion": "datafusion", + "duckdb": "duckdb", +} + +# The engines a target may be routed onto. +KNOWN_ENGINES: frozenset[str] = frozenset({"duckdb", "datafusion"}) + + +def scalar(value): + """Coerce an engine-specific scalar to a comparable plain Python value. + + ``None`` and any NaN (whether a ``float`` subclass or a numpy scalar that + only reveals its NaN-ness after ``.item()``) both collapse to ``None`` so + nullable integers from DataFusion compare equal to DuckDB's ``NULL`` and + can never reach the ``(is_none, value)`` sort key as a bare NaN. + """ + if value is None: + return None + # Fast path: a plain float NaN. + if isinstance(value, float) and math.isnan(value): + return None + if hasattr(value, "item"): + try: + value = value.item() + except (ValueError, AttributeError): + return value + # Re-check NaN AFTER .item(): a numpy NaN unwraps to a float NaN that + # must not survive as a sort key (Finding 6). + if value is None: + return None + if isinstance(value, float) and math.isnan(value): + return None + return value + + +def normalize(rows) -> list[tuple]: + """Return a row multiset as a sorted list of tuples for order-free compare. + + A list (not a set) preserves multiplicity so duplicate-row bugs surface; + sorting strips engine result ordering. Values are coerced to plain Python + scalars so DuckDB and DataFusion (pandas) rows compare equal. + """ + out = [tuple(scalar(v) for v in row) for row in rows] + return sorted(out, key=lambda r: tuple((v is None, v) for v in r)) + + +def resolve_routing(targets, engines=None) -> dict[str, tuple[str, str | None]]: + """Resolve each target to its ``(engine, transpile_dialect)`` pair. + + This is the pure routing decision lifted out of the oracle run loop so it + can be unit-tested without engines (Finding 7). + + Parameters + ---------- + targets : iterable of str + Target names to route (e.g. ``"generic"``, ``"datafusion"``, + ``"duckdb"``). + engines : dict[str, str] | None + Optional ``{target: engine}`` overrides for the default routing. Use + this to execute a target's portable SQL on a different engine (e.g. + route ``generic`` to DuckDB so a LATERAL operator can still be + compared). + + Returns + ------- + dict[str, tuple[str, str | None]] + ``{target: (engine, transpile_dialect)}`` for each requested target. + + Raises + ------ + ValueError + If a target name is not a known target, or an override routes to an + unknown engine. + """ + engine_for = dict(TARGET_ENGINE) + if engines: + for target, engine in engines.items(): + if target not in TARGET_ENGINE: + raise ValueError( + f"Unknown target {target!r} in engines override. " + f"Known targets: {sorted(TARGET_ENGINE)}." + ) + engine_for[target] = engine + + routing: dict[str, tuple[str, str | None]] = {} + for target in targets: + if target not in TARGET_ENGINE: + raise ValueError( + f"Unknown target {target!r}. Known targets: {sorted(TARGET_ENGINE)}." + ) + engine = engine_for[target] + if engine not in KNOWN_ENGINES: + raise ValueError( + f"Unknown engine {engine!r} for target {target!r}. " + f"Known engines: {sorted(KNOWN_ENGINES)}." + ) + routing[target] = (engine, TARGET_DIALECT[target]) + return routing + + +def assert_cross_target(results, expected, sql_by_target=None) -> None: + """Assert every target agrees with the others and with ``expected``. + + The comparison runs in two genuinely-reachable phases (Finding 1): + + 1. **Differential oracle** -- assert the engines agree *with each other*, + independent of ``expected``. With every engine's output already + collected, a disagreement yields a clear "engines disagree" message + naming both targets and showing both result sets. + 2. **Expectation check** -- assert the (now agreed) result equals the + expected multiset. + + Parameters + ---------- + results : dict[str, list[tuple]] + ``{target: normalized_rows}`` -- every target's already-normalized + result. Collected in full *before* any assertion so the first failing + target no longer short-circuits the cross-target comparison. + expected : list[tuple] + The normalized expected multiset. + sql_by_target : dict[str, str] | None + Optional ``{target: sql}`` for richer diagnostics. + """ + sql_by_target = sql_by_target or {} + run_targets = list(results) + + # Phase 1: engines must agree with each other (the differential oracle). + if len(run_targets) > 1: + reference = run_targets[0] + ref_rows = results[reference] + for other in run_targets[1:]: + assert results[other] == ref_rows, ( + f"engines disagree: target {reference!r} returned " + f"{ref_rows!r} but target {other!r} returned " + f"{results[other]!r}\n" + f"{reference} SQL: {sql_by_target.get(reference, '')}\n" + f"{other} SQL: {sql_by_target.get(other, '')}" + ) + + # Phase 2: the agreed result must equal the expectation. + for target in run_targets: + assert results[target] == expected, ( + f"target {target!r} returned {results[target]!r}, " + f"expected {expected!r}\n" + f"SQL: {sql_by_target.get(target, '')}" + ) + + +def register_record_batches(ctx, name, rows, schema, columns) -> None: + """Register ``rows`` as a record-batch table on a DataFusion context. + + The single pyarrow loader dance shared by :func:`run_datafusion` and the + #132 ``datafusion_ctx`` fixture (Finding 8). ``columns`` is the + ``(name, kind)`` schema; ``schema`` is the matching :class:`pyarrow.Schema`. + """ + import pyarrow as pa + + arrays = {col: [r[idx] for r in rows] for idx, (col, _kind) in enumerate(columns)} + ctx.register_record_batches(name, [pa.table(arrays, schema=schema).to_batches()]) + + +def arrow_schema(columns): + """Build a :class:`pyarrow.Schema` from a ``(name, kind)`` column spec.""" + import pyarrow as pa + + fields = [ + (col, pa.utf8() if kind == "utf8" else pa.int64()) for col, kind in columns + ] + return pa.schema(fields) + + +def run_duckdb(sql: str, table_data: dict, columns) -> list[tuple]: + """Register tables in an in-memory DuckDB and return normalized result rows.""" + import duckdb + + conn = duckdb.connect(":memory:") + try: + for name, rows in table_data.items(): + cols_ddl = ", ".join( + f'"{col}" {"VARCHAR" if kind == "utf8" else "BIGINT"}' + for col, kind in columns + ) + conn.execute(f"CREATE TABLE {name} ({cols_ddl})") + if rows: + placeholders = ", ".join("?" for _ in columns) + conn.executemany( + f"INSERT INTO {name} VALUES ({placeholders})", + [tuple(r) for r in rows], + ) + return normalize(conn.execute(sql).fetchall()) + finally: + conn.close() + + +def run_datafusion(sql: str, table_data: dict, columns) -> list[tuple]: + """Register tables in a DataFusion context and return normalized result rows.""" + from datafusion import SessionContext + + schema = arrow_schema(columns) + ctx = SessionContext() + for name, rows in table_data.items(): + register_record_batches(ctx, name, rows, schema, columns) + return normalize(ctx.sql(sql).to_pandas().itertuples(index=False, name=None)) + + +ENGINE_RUNNERS = { + "duckdb": run_duckdb, + "datafusion": run_datafusion, +} diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index afe4e6d..80bcdd7 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -21,6 +21,18 @@ query, the table data, and the expected rows. Cases where a target cannot run an operator yet (e.g. correlated ``LATERAL`` on DataFusion) restrict the run to the supported targets via the ``targets`` argument. + +The engine-free internals live in :mod:`tests.integration._oracle` (an +importable, non-test module) so they can be unit-tested without engines; this +fixture is a thin wrapper around them. + +This lane is deliberately kept an **explicit-case fixture**. It does NOT use +the SDLC guide's pairwise / Scenario-model / Hypothesis apparatus: each case is +a hand-written ``(query, data, expected)`` triple so a regression points at a +named operator scenario, and the cross-engine differential oracle -- not a +generator -- is what makes the cases load-bearing. The coordinate-encoding +sweep (``coordinate_space`` / #145) and strand (the schema here is +``chrom``/``start``/``end`` only) are out of scope by design. """ from __future__ import annotations @@ -30,6 +42,11 @@ from giql import Table from giql import transpile +from ._oracle import ENGINE_RUNNERS +from ._oracle import assert_cross_target +from ._oracle import normalize +from ._oracle import resolve_routing + # Default per-target schema: GIQL's default interval columns. Reserved words # (``end``) are quoted by the loaders below, mirroring ``test_binned_join.py``. DEFAULT_COLUMNS: tuple[tuple[str, str], ...] = ( @@ -38,104 +55,9 @@ ("end", "int64"), ) -# Target -> the engine the target's SQL is meant to execute on. -TARGET_ENGINE: dict[str, str] = { - "generic": "datafusion", - "datafusion": "datafusion", - "duckdb": "duckdb", -} - # Default target spread: every target executed on its intended engine. DEFAULT_TARGETS: tuple[str, ...] = ("generic", "datafusion", "duckdb") -# transpile() dialect argument per target name. -_TARGET_DIALECT: dict[str, str | None] = { - "generic": None, - "datafusion": "datafusion", - "duckdb": "duckdb", -} - - -def _normalize(rows) -> list[tuple]: - """Return a row multiset as a sorted list of tuples for order-free compare. - - A list (not a set) preserves multiplicity so duplicate-row bugs surface; - sorting strips engine result ordering. Values are coerced to plain Python - scalars so DuckDB and DataFusion (pandas) rows compare equal. - """ - out = [] - for row in rows: - out.append(tuple(_scalar(v) for v in row)) - return sorted(out, key=lambda r: tuple((v is None, v) for v in r)) - - -def _scalar(value): - """Coerce engine-specific scalars to comparable plain Python values.""" - if value is None: - return None - # pandas / numpy nullable integers and NaN surface from DataFusion. - try: - import math - - if isinstance(value, float) and math.isnan(value): - return None - except (TypeError, ValueError): - pass - if hasattr(value, "item"): - try: - return value.item() - except (ValueError, AttributeError): - return value - return value - - -def _run_duckdb(sql: str, table_data: dict, columns) -> list[tuple]: - """Register tables in an in-memory DuckDB and return normalized result rows.""" - import duckdb - - conn = duckdb.connect(":memory:") - try: - for name, rows in table_data.items(): - cols_ddl = ", ".join( - f'"{col}" {"VARCHAR" if kind == "utf8" else "BIGINT"}' - for col, kind in columns - ) - conn.execute(f"CREATE TABLE {name} ({cols_ddl})") - if rows: - placeholders = ", ".join("?" for _ in columns) - conn.executemany( - f"INSERT INTO {name} VALUES ({placeholders})", - [tuple(r) for r in rows], - ) - return _normalize(conn.execute(sql).fetchall()) - finally: - conn.close() - - -def _run_datafusion(sql: str, table_data: dict, columns) -> list[tuple]: - """Register tables in a DataFusion context and return normalized result rows.""" - import pyarrow as pa - from datafusion import SessionContext - - pa_fields = [ - (col, pa.utf8() if kind == "utf8" else pa.int64()) for col, kind in columns - ] - schema = pa.schema(pa_fields) - - ctx = SessionContext() - for name, rows in table_data.items(): - arrays = { - col: [r[idx] for r in rows] for idx, (col, _kind) in enumerate(columns) - } - ctx.register_record_batches(name, [pa.table(arrays, schema=schema).to_batches()]) - return _normalize(ctx.sql(sql).to_pandas().itertuples(index=False, name=None)) - - -_ENGINE_RUNNERS = { - "duckdb": _run_duckdb, - "datafusion": _run_datafusion, -} - @pytest.fixture def cross_target_oracle(): @@ -150,6 +72,7 @@ def cross_target_oracle(): tables=None, columns=DEFAULT_COLUMNS, targets=DEFAULT_TARGETS, + engines=None, **table_data, ) @@ -165,7 +88,8 @@ def cross_target_oracle(): ``columns`` The ``(name, kind)`` schema (``kind`` in ``{"utf8", "int64"}``) used to register ``table_data`` into both engines. Defaults to the GIQL default - ``chrom``/``start``/``end`` interval schema. + ``chrom``/``start``/``end`` interval schema, where ``chrom`` is ``utf8`` + and ``start``/``end`` are ``int64``. ``targets`` The target names to exercise. Defaults to all three. Restrict this when a target cannot run the operator on its engine yet (e.g. NEAREST's @@ -174,7 +98,7 @@ def cross_target_oracle(): Optional ``{target: engine}`` overrides for the default routing (``generic``/``datafusion`` -> DataFusion, ``duckdb`` -> DuckDB). Use this when a target's *portable* SQL must be executed on a different - engine for the case at hand — e.g. routing ``generic`` to DuckDB so a + engine for the case at hand -- e.g. routing ``generic`` to DuckDB so a LATERAL-based operator can still be compared against the duckdb target. ``**table_data`` ``name=[row, ...]`` table contents, where each row is a tuple matching @@ -197,39 +121,26 @@ def _oracle( if tables is None: tables = [_default_table(name, columns) for name in table_data] - engine_for = dict(TARGET_ENGINE) - if engines: - engine_for.update(engines) + routing = resolve_routing(targets, engines) - expected_rows = _normalize(expected) + expected_rows = normalize(expected) results: dict[str, list[tuple]] = {} + sql_by_target: dict[str, str] = {} + # Collect ALL target results first; defer every assertion to + # assert_cross_target so the first divergent target can no longer + # short-circuit the cross-target comparison (Finding 1). for target in targets: - engine = engine_for[target] - importorskip = pytest.importorskip - importorskip("duckdb" if engine == "duckdb" else "datafusion") + engine, dialect = routing[target] + pytest.importorskip("duckdb" if engine == "duckdb" else "datafusion") if engine == "datafusion": - importorskip("pyarrow") - - sql = transpile(query, tables=tables, dialect=_TARGET_DIALECT[target]) - rows = _ENGINE_RUNNERS[engine](sql, table_data, columns) - results[target] = rows - - assert rows == expected_rows, ( - f"target {target!r} on engine {engine!r} returned {rows!r}, " - f"expected {expected_rows!r}\nSQL: {sql}" - ) - - # Cross-target identity: every executed target agrees row-for-row. - run_targets = list(results) - if len(run_targets) > 1: - reference = run_targets[0] - for other in run_targets[1:]: - assert results[other] == results[reference], ( - f"targets {reference!r} and {other!r} disagree: " - f"{results[reference]!r} != {results[other]!r}" - ) + pytest.importorskip("pyarrow") + + sql = transpile(query, tables=tables, dialect=dialect) + sql_by_target[target] = sql + results[target] = ENGINE_RUNNERS[engine](sql, table_data, columns) + assert_cross_target(results, expected_rows, sql_by_target) return results return _oracle diff --git a/tests/integration/datafusion/conftest.py b/tests/integration/datafusion/conftest.py index f57d7bc..e6ce4f0 100644 --- a/tests/integration/datafusion/conftest.py +++ b/tests/integration/datafusion/conftest.py @@ -12,6 +12,12 @@ pytestmark = pytest.mark.integration +from .._oracle import arrow_schema # noqa: E402 +from .._oracle import register_record_batches # noqa: E402 + +# The default GIQL interval schema, shared with the cross-target oracle. +_INTERVAL_COLUMNS = (("chrom", "utf8"), ("start", "int64"), ("end", "int64")) + @pytest.fixture def datafusion_ctx(): @@ -21,29 +27,19 @@ def datafusion_ctx(): arguments and yields a DataFusion ``SessionContext`` with each table registered under the default ``chrom``/``start``/``end`` schema (matching the default :class:`giql.Table` column mapping). + + The pyarrow loader dance is the one shared helper in + :mod:`tests.integration._oracle` (Finding 8), so this fixture and the + cross-target oracle register tables identically. """ - import pyarrow as pa from datafusion import SessionContext - schema = pa.schema( - [ - ("chrom", pa.utf8()), - ("start", pa.int64()), - ("end", pa.int64()), - ] - ) + schema = arrow_schema(_INTERVAL_COLUMNS) def _build(**tables): ctx = SessionContext() for name, rows in tables.items(): - arrays = { - "chrom": [r[0] for r in rows], - "start": [r[1] for r in rows], - "end": [r[2] for r in rows], - } - ctx.register_record_batches( - name, [pa.table(arrays, schema=schema).to_batches()] - ) + register_record_batches(ctx, name, rows, schema, _INTERVAL_COLUMNS) return ctx return _build From 4282dc3d2055cf91182826c2b1d84fab565ec450 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 19:48:08 -0400 Subject: [PATCH 074/142] fix: Strict-xfail NEAREST oracle and clarify generic-vs-duckdb test --- .../datafusion/test_cross_target_oracle.py | 54 ++++++++++++------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/tests/integration/datafusion/test_cross_target_oracle.py b/tests/integration/datafusion/test_cross_target_oracle.py index 91f17be..03acdd3 100644 --- a/tests/integration/datafusion/test_cross_target_oracle.py +++ b/tests/integration/datafusion/test_cross_target_oracle.py @@ -154,21 +154,26 @@ def test_within_returns_enclosed_rows(self, cross_target_oracle): class TestCrossTargetOracleNearest: """Standalone NEAREST identity on the targets whose engine supports LATERAL.""" - def test_standalone_nearest_k1_agrees_on_duckdb_targets(self, cross_target_oracle): - """Test standalone NEAREST k=1 agrees across the LATERAL-capable targets. + def test_standalone_nearest_k1_agrees_generic_vs_duckdb_on_duckdb( + self, cross_target_oracle + ): + """Test NEAREST k=1 generic and duckdb SQL agree when both run on DuckDB. Given: A single-row peaks table and three candidate genes at varying distances on chr1. When: A correlated ``CROSS JOIN LATERAL NEAREST(..., k := 1)`` query runs - on the generic and duckdb targets, both executed on DuckDB. + for the generic and duckdb targets, both executed on DuckDB (the + generic target routed via ``engines={"generic": "duckdb"}``). Then: - Both targets should return the single nearest gene and agree. + The generic and duckdb SQL should both return the single nearest + gene and agree. - The generic target is routed to DuckDB here (not its default DataFusion - engine): the correlated LATERAL this expansion emits has no DataFusion - physical plan — the one documented cross-target gap. + This is a generic-vs-duckdb *equivalence* check, not a cross-target + identity check: DataFusion is never run here. The generic target is + routed to DuckDB because the correlated LATERAL this expansion emits has + no DataFusion physical plan — the one documented cross-target gap. """ # Arrange / Act / Assert cross_target_oracle( @@ -186,19 +191,28 @@ def test_standalone_nearest_k1_agrees_on_duckdb_targets(self, cross_target_oracl engines={"generic": "duckdb"}, ) - def test_nearest_on_datafusion_lateral_is_unsupported(self, cross_target_oracle): - """Test the NEAREST LATERAL expansion has no DataFusion physical plan. + @pytest.mark.xfail( + strict=True, + raises=Exception, + reason="DataFusion lacks correlated LATERAL (OuterReferenceColumn) — #142", + ) + def test_nearest_full_oracle_xfails_on_datafusion_lateral(self, cross_target_oracle): + """Test the full NEAREST oracle xfails on DataFusion's missing LATERAL. Given: - The same single-row NEAREST query. + The single-row NEAREST query and a candidate gene on chr1. When: - The oracle is asked to run only the datafusion target on DataFusion. + The oracle runs all three targets — the datafusion target executes + the correlated LATERAL on DataFusion, which has no physical plan. Then: - DataFusion should reject the correlated LATERAL, pinning the gap that - justifies excluding it from the identity assertion above. + DataFusion should raise its ``OuterReferenceColumn`` "not + implemented" error, pinning the gap as a strict xfail that + auto-promotes (XPASS -> fail) when #142 lands. An UNRELATED + DataFusion error must still fail loudly, so the match is narrowed + to the LATERAL signature before re-raising. """ # Arrange / Act - with pytest.raises(Exception) as excinfo: + try: cross_target_oracle( "SELECT a.chrom, a.start AS a_start, b.start AS b_start " "FROM peaks a " @@ -206,8 +220,12 @@ def test_nearest_on_datafusion_lateral_is_unsupported(self, cross_target_oracle) peaks=[("chr1", 200, 300)], genes=[("chr1", 280, 290)], expected=[("chr1", 200, 280)], - targets=("datafusion",), ) - - # Assert - assert "not implemented" in str(excinfo.value).lower() + except Exception as exc: # noqa: BLE001 + message = str(exc).lower() + # Assert: narrow the xfail to the documented LATERAL gap so any + # unrelated DataFusion failure escapes and fails the test loudly. + assert "outerreferencecolumn" in message or "not implemented" in message, ( + f"unexpected DataFusion error, not the LATERAL gap: {exc!r}" + ) + raise From e80c14139d2a100fd909ea3fa3c159a09d78ccea Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 19:56:32 -0400 Subject: [PATCH 075/142] test: Expand operator-expander registry and pass coverage --- tests/test_expander.py | 1154 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1154 insertions(+) diff --git a/tests/test_expander.py b/tests/test_expander.py index cc4667e..afa74af 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -30,6 +30,9 @@ from giql.expander import expand_operators from giql.expander import register from giql.expressions import GIQLDisjoin +from giql.expressions import GIQLMerge +from giql.expressions import Intersects +from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables @@ -60,6 +63,33 @@ def clean_registry(): REGISTRY._expanders.update(saved) +@pytest.fixture(autouse=True) +def _registry_leak_guard(): + """Assert the process-wide REGISTRY is empty at each test boundary. + + A leak guard: the registry is empty at import and must return to empty after + every test, so a test that registers without cleaning up (a leak that would + silently flip the no-op pass for a later test) fails loudly. Tests that + register on the process-wide REGISTRY do so through ``clean_registry``, which + clears on the way out; this guard catches anything that bypasses it. + """ + assert not REGISTRY._expanders, "REGISTRY leaked into a test from a prior one" + yield + assert not REGISTRY._expanders, "a test leaked a registration into REGISTRY" + + +class _CountingExpander: + """An expander recording each call so a walk's dispatch count can be pinned.""" + + def __init__(self, label="counted"): + self.calls = [] + self._label = label + + def __call__(self, node, ctx): + self.calls.append((node, ctx)) + return exp.Literal.string(self._label) + + class TestExpanderRegistry: """Tests for ExpanderRegistry registration and fallback resolution.""" @@ -257,6 +287,139 @@ def test_register_rejects_non_callable(self): registry.register(GenericTarget(), GIQLDisjoin, object()) +class TestExpanderRegistryFallbackGaps: + """Edge cases of the registry fallback and op-scoped keying.""" + + def test_non_generic_target_with_no_entries_resolves_none(self): + """Test that a target with neither a specific nor generic entry yields None. + + Given: + A registry with no entry for the target and no generic entry. + When: + Resolving (DataFusionTarget(), op). + Then: + It resolves to None (legacy emitter fallback), having exhausted the + chain. + """ + # Arrange + registry = ExpanderRegistry() + registry.register(DuckDBTarget(), GIQLDisjoin, _record("duckdb")) + + # Act + resolved = registry.resolve(DataFusionTarget(), GIQLDisjoin) + + # Assert + assert resolved is None + + def test_generic_entry_is_operator_scoped(self): + """Test that a generic entry for one operator does not satisfy another. + + Given: + A generic expander registered for operator X (GIQLDisjoin) only. + When: + Resolving operator Y (GIQLMerge) for a target. + Then: + It resolves to None — the generic fallback is keyed per operator, not + a catch-all for the target. + """ + # Arrange + registry = ExpanderRegistry() + registry.register(GenericTarget(), GIQLDisjoin, _record("generic")) + + # Act + resolved = registry.resolve(DuckDBTarget(), GIQLMerge) + + # Assert + assert resolved is None + + def test_contains_checks_exact_key_not_fallback(self): + """Test that __contains__ tests the exact key, ignoring the generic chain. + + Given: + A registry with a generic entry for op only. + When: + Testing membership of an exact (DuckDBTarget, op) key. + Then: + The exact target key reports absent while the generic key reports + present (membership does not walk the fallback chain). + """ + # Arrange + registry = ExpanderRegistry() + registry.register(GenericTarget(), GIQLDisjoin, _record("generic")) + + # Act & assert + assert (GenericTarget(), GIQLDisjoin) in registry + assert (DuckDBTarget(), GIQLDisjoin) not in registry + + def test_unregister_drops_entry_and_falls_through(self): + """Test that unregister drops an entry so the key resolves to None after. + + Given: + A registry with one (DuckDBTarget, op) entry. + When: + Unregistering that key. + Then: + Resolving it returns None and the key reports absent (public teardown + seam). + """ + # Arrange + registry = ExpanderRegistry() + registry.register(DuckDBTarget(), GIQLDisjoin, _record("duckdb")) + + # Act + registry.unregister(DuckDBTarget(), GIQLDisjoin) + + # Assert + assert registry.resolve(DuckDBTarget(), GIQLDisjoin) is None + assert (DuckDBTarget(), GIQLDisjoin) not in registry + + def test_unregister_absent_key_is_noop(self): + """Test that unregistering an absent key does not raise. + + Given: + An empty registry. + When: + Unregistering a key that was never registered. + Then: + It returns without raising (idempotent teardown). + """ + # Arrange + registry = ExpanderRegistry() + + # Act & assert (no raise) + registry.unregister(DuckDBTarget(), GIQLDisjoin) + assert (DuckDBTarget(), GIQLDisjoin) not in registry + + def test_public_teardown_seam_resets_process_registry(self): + """Test that the public seam tears a custom registration off REGISTRY. + + Given: + A custom expander registered on the process-wide REGISTRY via the + decorator (the public extension hook). + When: + Tearing it down through the public seam — unregister, then clear — + without touching private state. + Then: + The key resolves to None and REGISTRY is empty again, so the leak + guard's post-test assertion holds (the seam is a supported reset). + """ + + # Arrange + @register(DuckDBTarget, GIQLDisjoin) + def _expander(node, ctx): + return exp.column("x") + + assert (DuckDBTarget(), GIQLDisjoin) in REGISTRY + + # Act + REGISTRY.unregister(DuckDBTarget(), GIQLDisjoin) + REGISTRY.clear() + + # Assert + assert REGISTRY.resolve(DuckDBTarget(), GIQLDisjoin) is None + assert (DuckDBTarget(), GIQLDisjoin) not in REGISTRY + + class TestRegisterDecorator: """Tests for the @register extension-hook decorator.""" @@ -319,6 +482,154 @@ def _expander(node, ctx): # Assert assert clean_registry.resolve(DuckDBTarget(), GIQLDisjoin) is _expander + def test_register_class_and_instance_land_on_same_entry(self, clean_registry): + """Test that the class and an instance form key the same registry entry. + + Given: + A registration via the class form @register(DuckDBTarget, op). + When: + Registering again via the instance form @register(DuckDBTarget(), op). + Then: + The second lands on the same entry and overrides the first (the + decorator instantiates the class to a value-equal target key). + """ + + # Arrange + @register(DuckDBTarget, GIQLDisjoin) + def _first(node, ctx): + return exp.Literal.string("first") + + # Act + @register(DuckDBTarget(), GIQLDisjoin) + def _second(node, ctx): + return exp.Literal.string("second") + + # Assert + assert clean_registry.resolve(DuckDBTarget(), GIQLDisjoin) is _second + + def test_register_stacks_one_expander_for_two_targets(self, clean_registry): + """Test that stacking the decorator registers one expander for two targets. + + Given: + An expander decorated with @register for both DuckDB and DataFusion. + When: + Resolving each target's key. + Then: + Both resolve to the same single expander object. + """ + + # Arrange & act + @register(DuckDBTarget, GIQLDisjoin) + @register(DataFusionTarget, GIQLDisjoin) + def _shared(node, ctx): + return exp.Literal.string("shared") + + # Assert + assert clean_registry.resolve(DuckDBTarget(), GIQLDisjoin) is _shared + assert clean_registry.resolve(DataFusionTarget(), GIQLDisjoin) is _shared + + def test_register_overrides_through_decorator(self, clean_registry): + """Test that a second @register for one key overrides the first. + + Given: + Two expanders decorated for the same (target, op) key in order. + When: + Resolving the key. + Then: + It resolves to the second (last-write-wins through the decorator). + """ + + # Arrange + @register(GenericTarget, GIQLDisjoin) + def _first(node, ctx): + return exp.Literal.string("first") + + # Act + @register(GenericTarget, GIQLDisjoin) + def _second(node, ctx): + return exp.Literal.string("second") + + # Assert + assert clean_registry.resolve(GenericTarget(), GIQLDisjoin) is _second + + def test_register_accepts_object_form_through_decorator(self, clean_registry): + """Test that the decorator accepts an OperatorExpander object. + + Given: + An OperatorExpander object passed to the @register decorator. + When: + Resolving the key and invoking it. + Then: + It resolves to a callable delegating to the object's expand method. + """ + + # Arrange + class _Obj: + def expand(self, node, ctx): + return exp.Literal.string("obj-form") + + obj = _Obj() + + # Act + register(GenericTarget, GIQLDisjoin)(obj) + resolved = clean_registry.resolve(GenericTarget(), GIQLDisjoin) + + # Assert + assert resolved(None, None) == exp.Literal.string("obj-form") + + def test_register_rejects_non_callable_through_decorator(self, clean_registry): + """Test that decorating a non-callable, non-expander raises TypeError. + + Given: + A plain object that is neither callable nor an OperatorExpander. + When: + Passing it through the @register decorator. + Then: + It raises TypeError (the same guard as direct registration, proven + through the public decorator). + """ + # Arrange & act & assert + with pytest.raises(TypeError): + register(GenericTarget, GIQLDisjoin)(object()) + + def test_register_does_not_leak_across_targets(self, clean_registry): + """Test that registering for one target leaves another target unaffected. + + Given: + An expander registered for (DuckDBTarget, op) only. + When: + Resolving (DataFusionTarget, op), with no generic fallback present. + Then: + It resolves to None (no cross-target leakage). + """ + + # Arrange & act + @register(DuckDBTarget, GIQLDisjoin) + def _expander(node, ctx): + return exp.Literal.string("duck") + + # Assert + assert clean_registry.resolve(DataFusionTarget(), GIQLDisjoin) is None + + def test_register_returns_callable_decorated_function(self, clean_registry): + """Test that the bare decorated function stays callable standalone. + + Given: + A function decorated with @register. + When: + Calling the decorated function directly (outside the pass). + Then: + It runs and returns its expansion, unchanged by decoration. + """ + + # Arrange & act + @register(GenericTarget, GIQLDisjoin) + def _expander(node, ctx): + return exp.Literal.string("callable") + + # Assert + assert _expander(None, None) == exp.Literal.string("callable") + class TestExpansionContext: """Tests for the ExpansionContext value object.""" @@ -366,6 +677,124 @@ def test_alias_mints_unique_prefixed_names(self): assert first.startswith(EXPAND_ALIAS_PREFIX) assert second.startswith(EXPAND_ALIAS_PREFIX) + def test_resolution_none_is_tolerated(self): + """Test that a None resolution is a tolerated first-class context state. + + Given: + A context constructed with resolution=None. + When: + Reading its resolution attribute. + Then: + It is None without error (an un-annotated node is representable). + """ + # Arrange + ctx = ExpansionContext(exp.true(), None, GenericTarget(), Tables()) + + # Act & assert + assert ctx.resolution is None + + def test_tables_attribute_is_the_passed_container(self): + """Test that the context exposes the tables container it was built with. + + Given: + A context built with a specific Tables container. + When: + Reading its tables attribute. + Then: + It is that same container (identity), threaded for the expander. + """ + # Arrange + tables = _tables() + ctx = ExpansionContext(exp.true(), None, GenericTarget(), tables) + + # Act & assert + assert ctx.tables is tables + + def test_alias_uses_reserved_prefix_format(self): + """Test that minted aliases carry the reserved expander prefix. + + Given: + A fresh context. + When: + Minting an alias. + Then: + It starts with the EXPAND_ALIAS_PREFIX reserved namespace. + """ + # Arrange + ctx = ExpansionContext(exp.true(), None, GenericTarget(), Tables()) + + # Act + alias = ctx.alias() + + # Assert + assert alias.startswith(EXPAND_ALIAS_PREFIX) + + def test_alias_is_monotonic_within_one_context(self): + """Test that successive aliases in one context are all distinct. + + Given: + A single context. + When: + Minting several aliases. + Then: + They are all distinct (a monotonic sequence, no repeats). + """ + # Arrange + ctx = ExpansionContext(exp.true(), None, GenericTarget(), Tables()) + + # Act + names = [ctx.alias() for _ in range(5)] + + # Assert + assert len(set(names)) == 5 + + def test_alias_continues_across_contexts_sharing_a_sequence(self): + """Test that two contexts sharing one sequence mint non-colliding aliases. + + Given: + Two contexts built with the same shared name_sequence callable. + When: + Each mints an alias. + Then: + The aliases differ (the sequence threads continuity across the + per-node contexts of one run, preventing __giql_x_ collisions). + """ + # Arrange + from sqlglot.helper import name_sequence + + shared = name_sequence(EXPAND_ALIAS_PREFIX) + ctx_a = ExpansionContext(exp.true(), None, GenericTarget(), Tables(), shared) + ctx_b = ExpansionContext(exp.true(), None, GenericTarget(), Tables(), shared) + + # Act + a = ctx_a.alias() + b = ctx_b.alias() + + # Assert + assert a != b + + def test_capabilities_delegates_on_second_target(self): + """Test that capabilities delegates to a different target's capability set. + + Given: + A context built with a DataFusionTarget. + When: + Reading its capabilities property. + Then: + It is the DataFusion target's Capabilities (delegation is per-target, + not hardcoded to one engine). + """ + # Arrange + target = DataFusionTarget() + ctx = ExpansionContext(exp.true(), None, target, Tables()) + + # Act + caps = ctx.capabilities + + # Assert + assert caps is target.capabilities + assert caps.supports_lateral is False + # A fake custom target standing in for a user-defined engine: the extension-hook # seam must dispatch to it exactly as to the built-in targets. @@ -569,6 +998,731 @@ def test_transpile_sql_unchanged_with_pass_inert(self): assert EXPAND_ALIAS_PREFIX not in actual +# The nine GIQL operator expression classes the ExpandOperators pass inspects. +# Every one must ship opted out (GIQL_EXPAND=False) at this step so the pass is a +# strict no-op until a later migration flips one alongside its expander. +from giql.expressions import Contains # noqa: E402 +from giql.expressions import GIQLCluster # noqa: E402 +from giql.expressions import GIQLDistance # noqa: E402 +from giql.expressions import GIQLNearest # noqa: E402 +from giql.expressions import SpatialSetPredicate # noqa: E402 +from giql.expressions import Within # noqa: E402 + +_OPERATOR_CLASSES = ( + Intersects, + Contains, + Within, + SpatialSetPredicate, + GIQLDistance, + GIQLNearest, + GIQLDisjoin, + GIQLCluster, + GIQLMerge, +) + + +class TestOperatorOptOut: + """Every operator ships opted out of the ExpandOperators pass at this step.""" + + @pytest.mark.parametrize("operator", _OPERATOR_CLASSES, ids=lambda c: c.__name__) + def test_operator_class_ships_expand_disabled(self, operator): + """Test that each operator class ships GIQL_EXPAND=False. + + Given: + One of the nine GIQL operator expression classes. + When: + Reading its GIQL_EXPAND class attribute. + Then: + It should be False (no operator opts into expansion yet). + """ + # Arrange & act + flag = operator.GIQL_EXPAND + + # Assert + assert flag is False + + def test_expand_sentinel_is_false(self): + """Test that the shared _EXPAND opt-out sentinel is False. + + Given: + The expressions module's _EXPAND sentinel that every operator's + GIQL_EXPAND defaults to. + When: + Reading it. + Then: + It should be False, the single source the per-class flags inherit. + """ + # Arrange & act + from giql.expressions import _EXPAND + + # Assert + assert _EXPAND is False + + +class TestOptedInRestoresFlag: + """The _opted_in helper restores GIQL_EXPAND even when its body raises.""" + + def test_opted_in_restores_flag_after_exception(self): + """Test that _opted_in restores GIQL_EXPAND when the body raises. + + Given: + An operator class at its default GIQL_EXPAND=False. + When: + Its _opted_in body raises an exception. + Then: + The flag should be restored to False (the manager is exception-safe, + so a raising expansion test cannot leak an opt-in into a later test). + """ + # Arrange + assert GIQLMerge.GIQL_EXPAND is False + + # Act + with pytest.raises(RuntimeError): + with _opted_in(GIQLMerge): + assert GIQLMerge.GIQL_EXPAND is True + raise RuntimeError("boom") + + # Assert + assert GIQLMerge.GIQL_EXPAND is False + + +class TestIEJoinEarlyReturnSkipsExpansion: + """Pin Finding 2: the duckdb IEJoin early return skips the ExpandOperators pass.""" + + @pytest.mark.xfail( + strict=True, + reason="#141: the duckdb IEJoin early return in transpile() emits before " + "ExpandOperators runs, so a flagged operator on an IEJoin-eligible query " + "is not expanded. Flips to pass when #141 runs expansion before the " + "early return (or defers the IEJoin transformer to the registry).", + ) + def test_iejoin_query_expands_flagged_operator(self, clean_registry): + """Test that an IEJoin-eligible duckdb query expands a flagged operator. + + Given: + A column-to-column INTERSECTS join eligible for the duckdb IEJoin + path, with Intersects flagged GIQL_EXPAND and an expander registered. + When: + Transpiling with dialect='duckdb'. + Then: + The expander's sentinel should appear (currently it does NOT — the + IEJoin early return skips the pass; this xfail flips when #141 lands). + """ + # Arrange + clean_registry.register( + DuckDBTarget(), Intersects, lambda n, c: exp.column("__giql_iejoin_sentinel") + ) + query = ( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + with _opted_in(Intersects): + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "__giql_iejoin_sentinel" in sql + + def test_iejoin_query_emits_legacy_sql_unchanged(self, clean_registry): + """Test that the legacy IEJoin SQL is emitted regardless of a flagged op. + + Given: + The same IEJoin-eligible duckdb query with Intersects flagged and an + expander registered. + When: + Transpiling with dialect='duckdb'. + Then: + The legacy IEJoin SET VARIABLE SQL is emitted and the expander's + sentinel is absent (characterizing the current skip; the companion + xfail surfaces when #141 fixes it). + """ + # Arrange + clean_registry.register( + DuckDBTarget(), Intersects, lambda n, c: exp.column("__giql_iejoin_sentinel") + ) + query = ( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + with _opted_in(Intersects): + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + + # Assert + assert "SET VARIABLE __giql_iejoin_" in sql + assert "__giql_iejoin_sentinel" not in sql + + +class TestTranspileExpanderDispatch: + """Dispatch driven through the public transpile() entry point.""" + + def test_target_entry_dispatches_for_matching_dialect(self, clean_registry): + """Test that a (DuckDBTarget, op) entry dispatches for dialect='duckdb'. + + Given: + An expander registered for (DuckDBTarget, GIQLDisjoin) and the + operator flagged. + When: + Transpiling a DISJOIN query with dialect='duckdb'. + Then: + The expander's sentinel should reach the generated SQL. + """ + # Arrange + clean_registry.register( + DuckDBTarget(), GIQLDisjoin, lambda n, c: exp.column("DUCK_SENTINEL") + ) + + # Act + with _opted_in(GIQLDisjoin): + sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect="duckdb" + ) + + # Assert + assert "DUCK_SENTINEL" in sql + + @pytest.mark.parametrize("dialect", [None, "datafusion"]) + def test_target_entry_falls_through_for_other_dialects( + self, clean_registry, dialect + ): + """Test that a target-specific entry does not fire for other dialects. + + Given: + An expander registered only for (DuckDBTarget, GIQLDisjoin), flagged. + When: + Transpiling with dialect=None or 'datafusion'. + Then: + The sentinel should be absent and the legacy SQL emitted (the + target-specific entry does not resolve for a different target). + """ + # Arrange + clean_registry.register( + DuckDBTarget(), GIQLDisjoin, lambda n, c: exp.column("DUCK_SENTINEL") + ) + + # Act + with _opted_in(GIQLDisjoin): + sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=dialect + ) + + # Assert + assert "DUCK_SENTINEL" not in sql + + @pytest.mark.parametrize("dialect", [None, "duckdb", "datafusion"]) + def test_generic_entry_covers_all_dialects(self, clean_registry, dialect): + """Test that a (GenericTarget, op) entry dispatches via fallback everywhere. + + Given: + An expander registered only for (GenericTarget, GIQLDisjoin), flagged. + When: + Transpiling with each of the three dialects. + Then: + The generic expander's sentinel should reach the SQL in every case + (the fallback chain routes every target to the generic entry). + """ + # Arrange + clean_registry.register( + GenericTarget(), GIQLDisjoin, lambda n, c: exp.column("GENERIC_SENTINEL") + ) + + # Act + with _opted_in(GIQLDisjoin): + sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=dialect + ) + + # Assert + assert "GENERIC_SENTINEL" in sql + + def test_target_entry_shadows_generic_per_dialect(self, clean_registry): + """Test that a target entry shadows the generic entry for its dialect only. + + Given: + Both a (DuckDBTarget, op) and a (GenericTarget, op) expander, flagged. + When: + Transpiling with dialect='duckdb' versus dialect=None. + Then: + The duckdb path uses the target sentinel; the generic path uses the + generic sentinel (per-dialect shadowing through transpile()). + """ + # Arrange + clean_registry.register( + DuckDBTarget(), GIQLDisjoin, lambda n, c: exp.column("DUCK_SENTINEL") + ) + clean_registry.register( + GenericTarget(), GIQLDisjoin, lambda n, c: exp.column("GENERIC_SENTINEL") + ) + + # Act + with _opted_in(GIQLDisjoin): + duck_sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect="duckdb" + ) + generic_sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=None + ) + + # Assert + assert "DUCK_SENTINEL" in duck_sql and "GENERIC_SENTINEL" not in duck_sql + assert "GENERIC_SENTINEL" in generic_sql and "DUCK_SENTINEL" not in generic_sql + + def test_context_target_matches_resolved_dialect(self, clean_registry): + """Test that ctx.target equals resolve_target(dialect) at dispatch. + + Given: + A flagged DISJOIN and an expander capturing its context. + When: + Transpiling with dialect='datafusion'. + Then: + The captured ctx.target should equal resolve_target('datafusion'). + """ + # Arrange + from giql.targets import resolve_target + + captured = {} + + def _capture(node, ctx): + captured["target"] = ctx.target + return exp.column("ok") + + clean_registry.register(GenericTarget(), GIQLDisjoin, _capture) + + # Act + with _opted_in(GIQLDisjoin): + transpile( + "SELECT * FROM DISJOIN(variants)", + tables=["variants"], + dialect="datafusion", + ) + + # Assert + assert captured["target"] == resolve_target("datafusion") + + def test_throwing_expander_wraps_in_expansion_error(self, clean_registry): + """Test that an unexpected expander error is wrapped as an Expansion error. + + Given: + A flagged DISJOIN whose expander raises a non-ValueError. + When: + Transpiling. + Then: + transpile() should raise ValueError whose message starts with + 'Expansion error: ' (the stage boundary wraps unexpected errors). + """ + + # Arrange + def _boom(node, ctx): + raise RuntimeError("kaboom") + + clean_registry.register(GenericTarget(), GIQLDisjoin, _boom) + + # Act & assert + with _opted_in(GIQLDisjoin): + with pytest.raises(ValueError, match=r"^Expansion error: "): + transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) + + def test_value_error_from_expander_passes_verbatim(self, clean_registry): + """Test that a ValueError raised by an expander propagates unwrapped. + + Given: + A flagged DISJOIN whose expander raises a ValueError with a targeted + diagnostic. + When: + Transpiling. + Then: + The same ValueError message should propagate verbatim (the boundary + lets user-facing ValueErrors through untouched). + """ + + # Arrange + def _raise(node, ctx): + raise ValueError("a targeted diagnostic") + + clean_registry.register(GenericTarget(), GIQLDisjoin, _raise) + + # Act & assert + with _opted_in(GIQLDisjoin): + with pytest.raises(ValueError, match=r"^a targeted diagnostic$"): + transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) + + def test_expander_sees_canonicalized_operands(self, clean_registry): + """Test that expansion runs after canonicalization (pass ordering). + + Given: + A flagged DISJOIN with a non-canonical (1-based) target table, which + pass 2 wraps in a __giql_canon_ CTE, and an expander capturing its + node's resolution at dispatch. + When: + Transpiling. + Then: + The expander runs after canonicalization, so the surviving query + already carries the canonical wrapper CTE (expansion sees pass-2 + output, not the raw AST). + """ + # Arrange + captured = {} + + def _capture(node, ctx): + captured["resolution"] = ctx.resolution + # Returning the node unchanged leaves the canonical CTE in the tree. + return node + + clean_registry.register(GenericTarget(), GIQLDisjoin, _capture) + one_based = Table("variants", coordinate_system="1based") + + # Act + with _opted_in(GIQLDisjoin): + sql = transpile("SELECT * FROM DISJOIN(variants)", tables=[one_based]) + + # Assert + assert captured["resolution"] is not None + assert "__giql_canon_" in sql + + def test_replacement_reaches_generator(self, clean_registry): + """Test that the expander's replacement node is what the generator renders. + + Given: + A flagged DISJOIN and an expander returning a distinctive sentinel. + When: + Transpiling. + Then: + The sentinel appears in the final SQL and no DISJOIN remains, so the + replacement reached the generator (end-to-end). + """ + # Arrange + clean_registry.register( + GenericTarget(), GIQLDisjoin, lambda n, c: exp.column("REACHED_GENERATOR") + ) + + # Act + with _opted_in(GIQLDisjoin): + sql = transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) + + # Assert + assert "REACHED_GENERATOR" in sql + assert "DISJOIN" not in sql.upper() + + def test_unknown_dialect_raises_before_dispatch(self, clean_registry): + """Test that an unknown dialect raises before any expander dispatches. + + Given: + A flagged DISJOIN and a generic expander that would record a call. + When: + Transpiling with an unrecognized dialect. + Then: + transpile() raises ValueError for the unknown dialect and the + expander never runs (dialect resolution precedes dispatch). + """ + # Arrange + counting = _CountingExpander() + clean_registry.register(GenericTarget(), GIQLDisjoin, counting) + + # Act & assert + with _opted_in(GIQLDisjoin): + with pytest.raises(ValueError, match="Unknown dialect"): + transpile( + "SELECT * FROM DISJOIN(variants)", + tables=["variants"], + dialect="postgres", + ) + assert counting.calls == [] + + +class TestExpandOperatorsWalk: + """Real-AST walk behavior of the ExpandOperators pass.""" + + def test_walk_visits_every_sibling_operator(self, clean_registry): + """Test that the pass dispatches once per sibling operator and clears them. + + Given: + A query with two sibling DISJOIN operators, flagged, and a counting + expander. + When: + Running the pass. + Then: + The expander is called exactly twice and no DISJOIN node remains. + """ + # Arrange + counting = _CountingExpander() + clean_registry.register(GenericTarget(), GIQLDisjoin, counting) + tables = _tables(("variants", "genes")) + ast = _prepare( + "SELECT * FROM DISJOIN(variants) UNION ALL SELECT * FROM DISJOIN(genes)", + tables, + ) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + result = pass_.transform(ast) + + # Assert + assert len(counting.calls) == 2 + assert not list(result.find_all(GIQLDisjoin)) + + def test_walk_reaches_nested_operator(self, clean_registry): + """Test that the walk reaches an operator nested in a derived table. + + Given: + A DISJOIN nested inside a derived table, flagged, with a counting + expander. + When: + Running the pass. + Then: + The nested operator is dispatched exactly once. + """ + # Arrange + counting = _CountingExpander() + clean_registry.register(GenericTarget(), GIQLDisjoin, counting) + tables = _tables() + ast = _prepare("SELECT * FROM (SELECT * FROM DISJOIN(variants)) t", tables) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + pass_.transform(ast) + + # Assert + assert len(counting.calls) == 1 + + @pytest.mark.parametrize( + "query", + [ + "SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1-100'", + "SELECT * FROM peaks a JOIN genes b ON a.interval INTERSECTS 'chr1:1-100'", + "SELECT * FROM (SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1-100') t", + "WITH d AS (SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1-100') " + "SELECT * FROM d", + ], + ids=["where", "join_on", "subquery", "cte"], + ) + def test_walk_reaches_operator_in_each_location(self, clean_registry, query): + """Test that the walk reaches an operator in any clause location. + + Given: + An INTERSECTS predicate placed in a WHERE, JOIN-ON, subquery, or CTE, + flagged, with a counting expander. + When: + Running the pass. + Then: + The operator is dispatched exactly once regardless of location. + """ + # Arrange + counting = _CountingExpander() + clean_registry.register(GenericTarget(), Intersects, counting) + tables = _tables(("peaks", "genes")) + ast = _prepare(query, tables) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(Intersects): + pass_.transform(ast) + + # Assert + assert len(counting.calls) == 1 + + def test_walk_replacement_serializes_through_generator(self, clean_registry): + """Test that the pass's replacement serializes cleanly to SQL. + + Given: + A flagged DISJOIN and an expander returning a column sentinel. + When: + Running the pass and serializing the result with BaseGIQLGenerator. + Then: + The sentinel appears in the serialized SQL (replacement integrity). + """ + # Arrange + clean_registry.register( + GenericTarget(), GIQLDisjoin, lambda n, c: exp.column("WALK_SENTINEL") + ) + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + result = pass_.transform(ast) + sql = BaseGIQLGenerator(tables=tables).generate(result) + + # Assert + assert "WALK_SENTINEL" in sql + + def test_walk_identity_return_leaves_node_in_place(self, clean_registry): + """Test that an expander returning the node itself triggers no replacement. + + Given: + A flagged DISJOIN whose expander returns the node unchanged. + When: + Running the pass. + Then: + The DISJOIN node remains in the tree (the identity-return guard skips + the replace call). + """ + # Arrange + clean_registry.register(GenericTarget(), GIQLDisjoin, lambda n, c: n) + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + result = pass_.transform(ast) + + # Assert + assert list(result.find_all(GIQLDisjoin)) + + def test_walk_dispatches_mixed_operator_types_per_type(self, clean_registry): + """Test that the walk dispatches each operator type to its own expander. + + Given: + A query containing both a DISJOIN and an INTERSECTS, both flagged, + each with its own counting expander. + When: + Running the pass. + Then: + Each expander is called once, dispatched by operator type. + """ + # Arrange + disjoin_counter = _CountingExpander("disjoin") + intersects_counter = _CountingExpander("intersects") + clean_registry.register(GenericTarget(), GIQLDisjoin, disjoin_counter) + clean_registry.register(GenericTarget(), Intersects, intersects_counter) + tables = _tables(("variants", "peaks")) + ast = _prepare( + "SELECT * FROM DISJOIN(variants) " + "WHERE EXISTS (SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1-100')", + tables, + ) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + with _opted_in(Intersects): + pass_.transform(ast) + + # Assert + assert len(disjoin_counter.calls) == 1 + assert len(intersects_counter.calls) == 1 + + def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): + """Test that only the flagged operator type is replaced when both registered. + + Given: + A DISJOIN and an INTERSECTS, both with registered expanders, but only + INTERSECTS flagged GIQL_EXPAND. + When: + Running the pass. + Then: + The INTERSECTS is replaced while the DISJOIN node remains (the gate is + per-type). + """ + # Arrange + clean_registry.register( + GenericTarget(), GIQLDisjoin, lambda n, c: exp.column("DJ") + ) + clean_registry.register( + GenericTarget(), Intersects, lambda n, c: exp.column("IX") + ) + tables = _tables(("variants", "peaks")) + ast = _prepare( + "SELECT * FROM DISJOIN(variants) " + "WHERE EXISTS (SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1-100')", + tables, + ) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(Intersects): + result = pass_.transform(ast) + + # Assert + assert list(result.find_all(GIQLDisjoin)) + assert not list(result.find_all(Intersects)) + + def test_walk_shares_alias_sequence_across_sibling_expanders(self, clean_registry): + """Test that sibling expanders draw from one alias sequence (no collision). + + Given: + Two sibling DISJOIN operators, flagged, with an expander that mints an + alias per call and records it. + When: + Running the pass. + Then: + The two minted aliases differ (the run threads a single + name_sequence across every per-node context, so no __giql_x_ + collision). + """ + # Arrange + seen = [] + + def _mint(node, ctx): + seen.append(ctx.alias()) + return exp.column("c") + + clean_registry.register(GenericTarget(), GIQLDisjoin, _mint) + tables = _tables(("variants", "genes")) + ast = _prepare( + "SELECT * FROM DISJOIN(variants) UNION ALL SELECT * FROM DISJOIN(genes)", + tables, + ) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + pass_.transform(ast) + + # Assert + assert len(seen) == 2 + assert seen[0] != seen[1] + assert all(a.startswith(EXPAND_ALIAS_PREFIX) for a in seen) + + def test_lazy_import_has_no_cycle_in_fresh_process(self): + """Test that importing the expander module standalone raises no cycle. + + Given: + A fresh Python process with nothing pre-imported. + When: + Importing giql.expander and calling its lazy operator accessor. + Then: + It imports and resolves the nine operator classes without an import + cycle (the lazy _giql_operators import breaks the module cycle). + """ + # Arrange + import os + import subprocess + import sys + + code = ( + "import giql.expander as e; " + "ops = e._giql_operators(); " + "assert len(ops) == 9, ops; " + "print('ok')" + ) + # Strip coverage's subprocess auto-start hooks so the child is a genuinely + # clean interpreter exercising only the import (not the parent run's + # coverage instrumentation, which is irrelevant to the cycle check). + env = { + k: v + for k, v in os.environ.items() + if not k.startswith("COV_CORE") and k != "COVERAGE_PROCESS_START" + } + + # Act + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env, + ) + + # Assert + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "ok" + + def _tables(names=("variants",)) -> Tables: """Build a Tables container registering each name with default encoding.""" container = Tables() From 363a88ddc49cb843e275f50159542c98a85d0023 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 20:03:37 -0400 Subject: [PATCH 076/142] fix: Synthesize empty record batch for DataFusion empty tables --- tests/integration/_oracle.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/integration/_oracle.py b/tests/integration/_oracle.py index d6634e7..f0bd377 100644 --- a/tests/integration/_oracle.py +++ b/tests/integration/_oracle.py @@ -188,11 +188,22 @@ def register_record_batches(ctx, name, rows, schema, columns) -> None: The single pyarrow loader dance shared by :func:`run_datafusion` and the #132 ``datafusion_ctx`` fixture (Finding 8). ``columns`` is the ``(name, kind)`` schema; ``schema`` is the matching :class:`pyarrow.Schema`. + + Empty ``rows`` need care: ``pa.table(...).to_batches()`` yields an EMPTY + batch *list* for zero rows, and ``SessionContext.register_record_batches`` + PANICS on an empty partition (``index out of bounds`` in DataFusion's Rust + core). DuckDB sidesteps this with an ``if rows`` guard in :func:`run_duckdb`; + DataFusion has no such guard, so this helper synthesizes a single empty + record batch with the declared schema instead. This keeps empty-input cases + runnable on DataFusion (T4 pins both the panic and this guard). """ import pyarrow as pa arrays = {col: [r[idx] for r in rows] for idx, (col, _kind) in enumerate(columns)} - ctx.register_record_batches(name, [pa.table(arrays, schema=schema).to_batches()]) + batches = pa.table(arrays, schema=schema).to_batches() + if not batches: + batches = [pa.RecordBatch.from_pylist([], schema=schema)] + ctx.register_record_batches(name, [batches]) def arrow_schema(columns): From 7be0745e09817d46058bcb223298cc64a651cff8 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 20:03:37 -0400 Subject: [PATCH 077/142] test: Add engine-free unit tests for oracle internals --- tests/integration/test_oracle_internals.py | 477 +++++++++++++++++++++ 1 file changed, 477 insertions(+) create mode 100644 tests/integration/test_oracle_internals.py diff --git a/tests/integration/test_oracle_internals.py b/tests/integration/test_oracle_internals.py new file mode 100644 index 0000000..ceb4398 --- /dev/null +++ b/tests/integration/test_oracle_internals.py @@ -0,0 +1,477 @@ +"""Engine-free unit tests for the cross-target oracle internals (#139, T1). + +These prove the oracle *itself* is sound before any engine runs: that +:func:`normalize` / :func:`scalar` collapse engine quirks without hiding real +differences, that :func:`assert_cross_target` genuinely raises on a wrong +``expected`` and on engine disagreement, and that :func:`resolve_routing` maps +and guards targets/engines. No DuckDB / DataFusion / pyarrow needed, so they +run on every machine and localize oracle bugs away from engine bugs. +""" + +import math + +import pytest + +from tests.integration._oracle import assert_cross_target +from tests.integration._oracle import normalize +from tests.integration._oracle import resolve_routing +from tests.integration._oracle import scalar + + +class _FakeNumpyScalar: + """A scalar that only reveals its value (and NaN-ness) after ``.item()``.""" + + def __init__(self, value): + self._value = value + + def item(self): + return self._value + + +class TestScalar: + """`scalar` coercion of engine-specific values.""" + + def test_scalar_none_returns_none(self): + """Test None passes through as None. + + Given: + A None value. + When: + scalar() coerces it. + Then: + It should return None. + """ + # Arrange / Act / Assert + assert scalar(None) is None + + def test_scalar_plain_float_nan_returns_none(self): + """Test a plain float NaN collapses to None. + + Given: + A bare float NaN as a DuckDB/DataFusion null surrogate. + When: + scalar() coerces it. + Then: + It should return None so it compares equal to a real NULL. + """ + # Arrange / Act / Assert + assert scalar(float("nan")) is None + + def test_scalar_numpy_nan_after_item_returns_none(self): + """Test a NaN that only surfaces after .item() collapses to None. + + Given: + A numpy-like scalar whose .item() unwraps to a float NaN. + When: + scalar() coerces it and re-checks NaN AFTER .item() (Finding 6). + Then: + It should return None, never a bare NaN sort key. + """ + # Arrange + value = _FakeNumpyScalar(float("nan")) + + # Act + result = scalar(value) + + # Assert + assert result is None + + def test_scalar_numpy_int_returns_plain_int(self): + """Test a numpy-like int scalar unwraps to a plain Python int. + + Given: + A numpy-like scalar wrapping an int. + When: + scalar() coerces it. + Then: + It should return the plain int value. + """ + # Arrange / Act + result = scalar(_FakeNumpyScalar(42)) + + # Assert + assert result == 42 + assert isinstance(result, int) + + +class TestNormalize: + """`normalize` multiset construction.""" + + def test_normalize_sorts_rows_order_free(self): + """Test normalize returns a sorted, order-free row list. + + Given: + Two rows supplied in descending order. + When: + normalize() builds the multiset. + Then: + The result should be sorted ascending regardless of input order. + """ + # Arrange / Act + result = normalize([("chr2", 5, 6), ("chr1", 1, 2)]) + + # Assert + assert result == [("chr1", 1, 2), ("chr2", 5, 6)] + + def test_normalize_preserves_multiplicity(self): + """Test normalize keeps duplicate rows (list, not set). + + Given: + Two identical rows. + When: + normalize() builds the multiset. + Then: + Both copies should survive so duplicate-row bugs surface. + """ + # Arrange / Act + result = normalize([("chr1", 1, 2), ("chr1", 1, 2)]) + + # Assert + assert result == [("chr1", 1, 2), ("chr1", 1, 2)] + + def test_normalize_is_determinor_under_shuffle(self): + """Test normalize yields an identical result under row shuffling. + + Given: + A row multiset and a reversed copy of it. + When: + normalize() is applied to both orderings. + Then: + The two normalized results should be identical (determinism + self-test), including for None-bearing rows. + """ + # Arrange + rows = [ + ("chr1", 1, 2), + ("chr1", None, 9), + ("chr2", 3, 4), + ("chr1", 1, 2), + ] + + # Act + forward = normalize(rows) + reverse = normalize(list(reversed(rows))) + + # Assert + assert forward == reverse + + def test_normalize_treats_nan_and_none_alike(self): + """Test a NaN and a None sort into the same normalized position. + + Given: + One row with a None and one with a float NaN in the same column. + When: + normalize() coerces both. + Then: + They should compare equal so engine null surrogates do not diverge. + """ + # Arrange / Act + with_none = normalize([("chr1", None, 2)]) + with_nan = normalize([("chr1", math.nan, 2)]) + + # Assert + assert with_none == with_nan + + +class TestAssertCrossTarget: + """`assert_cross_target` differential + expectation phases.""" + + def test_passes_when_targets_agree_and_match_expected(self): + """Test no error when every target agrees and equals expected. + + Given: + Two targets returning the same normalized rows and a matching + expected multiset. + When: + assert_cross_target() runs. + Then: + It should not raise. + """ + # Arrange + rows = normalize([("chr1", 1, 2)]) + results = {"generic": rows, "duckdb": rows} + + # Act / Assert + assert_cross_target(results, normalize([("chr1", 1, 2)])) + + def test_raises_on_value_difference_from_expected(self): + """Test a value diff from expected raises in the expectation phase. + + Given: + Agreeing targets whose value differs from expected. + When: + assert_cross_target() runs. + Then: + It should raise AssertionError naming the expectation mismatch. + """ + # Arrange + rows = normalize([("chr1", 1, 2)]) + results = {"generic": rows, "duckdb": rows} + + # Act / Assert + with pytest.raises(AssertionError): + assert_cross_target(results, normalize([("chr1", 1, 99)])) + + def test_raises_on_extra_row_versus_expected(self): + """Test an extra row over expected raises. + + Given: + Agreeing targets returning one more row than expected. + When: + assert_cross_target() runs. + Then: + It should raise AssertionError. + """ + # Arrange + rows = normalize([("chr1", 1, 2), ("chr1", 3, 4)]) + results = {"generic": rows, "duckdb": rows} + + # Act / Assert + with pytest.raises(AssertionError): + assert_cross_target(results, normalize([("chr1", 1, 2)])) + + def test_raises_on_missing_row_versus_expected(self): + """Test a missing row versus expected raises. + + Given: + Agreeing targets returning one fewer row than expected. + When: + assert_cross_target() runs. + Then: + It should raise AssertionError. + """ + # Arrange + rows = normalize([("chr1", 1, 2)]) + results = {"generic": rows, "duckdb": rows} + + # Act / Assert + with pytest.raises(AssertionError): + assert_cross_target(results, normalize([("chr1", 1, 2), ("chr1", 3, 4)])) + + def test_raises_on_empty_versus_nonempty_expected(self): + """Test an empty result against a non-empty expectation raises. + + Given: + Agreeing targets returning zero rows but a non-empty expected set. + When: + assert_cross_target() runs. + Then: + It should raise AssertionError. + """ + # Arrange + results = {"generic": [], "duckdb": []} + + # Act / Assert + with pytest.raises(AssertionError): + assert_cross_target(results, normalize([("chr1", 1, 2)])) + + def test_raises_on_nonempty_versus_empty_expected(self): + """Test a non-empty result against an empty expectation raises. + + Given: + Agreeing targets returning a row but an empty expected set. + When: + assert_cross_target() runs. + Then: + It should raise AssertionError. + """ + # Arrange + rows = normalize([("chr1", 1, 2)]) + results = {"generic": rows, "duckdb": rows} + + # Act / Assert + with pytest.raises(AssertionError): + assert_cross_target(results, []) + + def test_raises_on_multiset_multiplicity_divergence(self): + """Test differing duplicate counts raise (list-not-set semantics). + + Given: + Agreeing targets returning a row twice but expected only once. + When: + assert_cross_target() runs. + Then: + It should raise because multiplicity is preserved, not collapsed. + """ + # Arrange + rows = normalize([("chr1", 1, 2), ("chr1", 1, 2)]) + results = {"generic": rows, "duckdb": rows} + + # Act / Assert + with pytest.raises(AssertionError): + assert_cross_target(results, normalize([("chr1", 1, 2)])) + + def test_raises_on_none_versus_value_difference(self): + """Test a None where expected holds a value raises. + + Given: + Agreeing targets with a None column but expected with a real value. + When: + assert_cross_target() runs. + Then: + It should raise: None and a value are genuinely distinct. + """ + # Arrange + rows = normalize([("chr1", None, 2)]) + results = {"generic": rows, "duckdb": rows} + + # Act / Assert + with pytest.raises(AssertionError): + assert_cross_target(results, normalize([("chr1", 5, 2)])) + + def test_coercion_normalizes_type_but_preserves_value_diff(self): + """Test type coercion does not mask a genuine value difference. + + Given: + Targets whose values are numpy-like scalars equal to expected ints + in one case and differing in another. + When: + assert_cross_target() compares the coerced multisets. + Then: + The equal case passes and the differing case raises — coercion + normalizes type without hiding a real difference. + """ + # Arrange + equal_rows = normalize([("chr1", _FakeNumpyScalar(1), _FakeNumpyScalar(2))]) + diff_rows = normalize([("chr1", _FakeNumpyScalar(1), _FakeNumpyScalar(2))]) + + # Act / Assert + assert_cross_target( + {"generic": equal_rows, "duckdb": equal_rows}, normalize([("chr1", 1, 2)]) + ) + with pytest.raises(AssertionError): + assert_cross_target( + {"generic": diff_rows, "duckdb": diff_rows}, + normalize([("chr1", 1, 99)]), + ) + + def test_raises_with_engines_disagree_message(self): + """Test two disagreeing engines trigger the differential assertion. + + Given: + A results dict where the duckdb target returns a different row from + the generic target, both matching neither each other. + When: + assert_cross_target() runs (now reachable after Finding 1). + Then: + It should raise AssertionError whose message says the engines + disagree and names both targets — independent of expected. + """ + # Arrange + generic_rows = normalize([("chr1", 1, 2)]) + duckdb_rows = normalize([("chr1", 1, 3)]) + results = {"generic": generic_rows, "duckdb": duckdb_rows} + + # Act + with pytest.raises(AssertionError) as excinfo: + assert_cross_target(results, generic_rows) + + # Assert + message = str(excinfo.value) + assert "engines disagree" in message + assert "generic" in message and "duckdb" in message + + def test_disagreement_checked_before_expectation(self): + """Test the differential phase fires even when one target matches expected. + + Given: + A generic target equal to expected and a duckdb target that differs, + so a mid-loop ``== expected`` design would have passed generic and + never compared the engines. + When: + assert_cross_target() runs. + Then: + It should raise the engines-disagree error first, proving Finding 1 + made the cross-target assertion genuinely reachable. + """ + # Arrange + expected = normalize([("chr1", 1, 2)]) + results = {"generic": expected, "duckdb": normalize([("chr1", 9, 9)])} + + # Act / Assert + with pytest.raises(AssertionError, match="engines disagree"): + assert_cross_target(results, expected) + + +class TestResolveRouting: + """`resolve_routing` default map, overrides, and guards.""" + + def test_default_routing_maps_each_target_to_its_engine(self): + """Test the default routing maps targets to their intended engines. + + Given: + All three default targets and no overrides. + When: + resolve_routing() resolves them. + Then: + generic/datafusion should route to DataFusion and duckdb to DuckDB, + each carrying its transpile dialect. + """ + # Arrange / Act + routing = resolve_routing(("generic", "datafusion", "duckdb")) + + # Assert + assert routing == { + "generic": ("datafusion", None), + "datafusion": ("datafusion", "datafusion"), + "duckdb": ("duckdb", "duckdb"), + } + + def test_engines_override_reroutes_target(self): + """Test an engines override reroutes a target's engine but not its dialect. + + Given: + The generic and duckdb targets with generic routed to duckdb. + When: + resolve_routing() applies the override. + Then: + generic should execute on duckdb while keeping its None dialect. + """ + # Arrange / Act + routing = resolve_routing(("generic", "duckdb"), engines={"generic": "duckdb"}) + + # Assert + assert routing["generic"] == ("duckdb", None) + assert routing["duckdb"] == ("duckdb", "duckdb") + + def test_unknown_target_raises_value_error(self): + """Test an unknown target name raises a clear ValueError (Finding 7). + + Given: + A target name that is not a registered target. + When: + resolve_routing() resolves it. + Then: + It should raise ValueError, not a bare KeyError. + """ + # Arrange / Act / Assert + with pytest.raises(ValueError, match="Unknown target"): + resolve_routing(("postgres",)) + + def test_unknown_engine_override_raises_value_error(self): + """Test an override to an unknown engine raises a clear ValueError. + + Given: + A valid target rerouted to an engine the oracle cannot run. + When: + resolve_routing() resolves it. + Then: + It should raise ValueError naming the unknown engine. + """ + # Arrange / Act / Assert + with pytest.raises(ValueError, match="Unknown engine"): + resolve_routing(("generic",), engines={"generic": "spark"}) + + def test_unknown_target_in_override_raises_value_error(self): + """Test an override keyed on an unknown target raises a clear ValueError. + + Given: + An engines override naming a target that does not exist. + When: + resolve_routing() resolves it. + Then: + It should raise ValueError naming the unknown override target. + """ + # Arrange / Act / Assert + with pytest.raises(ValueError, match="Unknown target"): + resolve_routing(("generic",), engines={"nope": "duckdb"}) From c166bf836d736291ed16986dae8d54507c3cd0e7 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 20:03:37 -0400 Subject: [PATCH 078/142] test: Cover operator breadth, runners, and lane guards --- .../datafusion/test_cross_target_oracle.py | 432 +++++++++++++++++- .../datafusion/test_oracle_runners.py | 275 +++++++++++ tests/integration/test_oracle_lane.py | 208 +++++++++ 3 files changed, 913 insertions(+), 2 deletions(-) create mode 100644 tests/integration/datafusion/test_oracle_runners.py create mode 100644 tests/integration/test_oracle_lane.py diff --git a/tests/integration/datafusion/test_cross_target_oracle.py b/tests/integration/datafusion/test_cross_target_oracle.py index 03acdd3..5cffd69 100644 --- a/tests/integration/datafusion/test_cross_target_oracle.py +++ b/tests/integration/datafusion/test_cross_target_oracle.py @@ -8,8 +8,10 @@ every later migration (#140-#144) will consume. NEAREST's expansion uses a correlated ``LATERAL`` subquery, which DataFusion has -no physical plan for today; its case therefore restricts the oracle to the -DuckDB-executed targets and is the one documented cross-target gap. +no physical plan for today; its generic-vs-duckdb equivalence case runs both on +DuckDB, and the full three-target oracle is pinned as a strict xfail (#142) that +auto-promotes when DataFusion gains correlated LATERAL — the one documented +cross-target gap. """ import pytest @@ -18,6 +20,8 @@ pytest.importorskip("datafusion") pytest.importorskip("pyarrow") +from giql import Table # noqa: E402 + pytestmark = pytest.mark.integration @@ -229,3 +233,427 @@ def test_nearest_full_oracle_xfails_on_datafusion_lateral(self, cross_target_ora f"unexpected DataFusion error, not the LATERAL gap: {exc!r}" ) raise + + +class TestCrossTargetOracleIntersectsAnyAll: + """INTERSECTS ANY/ALL identity across all targets (T2).""" + + def test_intersects_any_returns_rows_matching_either_range( + self, cross_target_oracle + ): + """Test INTERSECTS ANY returns rows overlapping at least one range. + + Given: + Peaks overlapping the first range, the second range, neither, and a + different chromosome. + When: + An INTERSECTS ANY query over two ranges runs on every target. + Then: + Every target should return exactly the two peaks overlapping a + listed range, in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS ANY('chr1:1000-2000', 'chr1:5000-6000')", + peaks=[ + ("chr1", 1500, 1800), + ("chr1", 5500, 5600), + ("chr1", 3000, 3100), + ("chr2", 1500, 1800), + ], + expected=[("chr1", 1500, 1800), ("chr1", 5500, 5600)], + ) + + def test_intersects_any_no_match_returns_zero_rows(self, cross_target_oracle): + """Test INTERSECTS ANY with no overlapping peak returns zero rows. + + Given: + A peak overlapping neither listed range. + When: + The INTERSECTS ANY query runs on every target. + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS ANY('chr1:1000-2000', 'chr1:5000-6000')", + peaks=[("chr1", 8000, 8100)], + expected=[], + ) + + def test_intersects_all_returns_rows_matching_every_range(self, cross_target_oracle): + """Test INTERSECTS ALL returns rows overlapping every listed range. + + Given: + One peak overlapping both ranges and one overlapping neither. + When: + An INTERSECTS ALL query over two ranges runs on every target. + Then: + Every target should return only the peak overlapping both ranges. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS ALL('chr1:1000-2000', 'chr1:1500-1700')", + peaks=[("chr1", 1400, 1800), ("chr1", 100, 200)], + expected=[("chr1", 1400, 1800)], + ) + + def test_intersects_all_no_full_match_returns_zero_rows(self, cross_target_oracle): + """Test INTERSECTS ALL returns zero rows when no peak overlaps all ranges. + + Given: + Peaks each overlapping only one of the two ranges. + When: + The INTERSECTS ALL query runs on every target. + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS ALL('chr1:1000-1200', 'chr1:5000-6000')", + peaks=[("chr1", 1050, 1100), ("chr1", 5050, 5100)], + expected=[], + ) + + +class TestCrossTargetOracleDistance: + """DISTANCE column-to-column identity across all targets (T2).""" + + def test_distance_column_to_column_filters_near_pairs(self, cross_target_oracle): + """Test a column-to-column DISTANCE predicate agrees across targets. + + Given: + A peak and two genes: one within the distance threshold and one far + beyond it on the same chromosome. + When: + A DISTANCE(a, b) < threshold join runs on every target. + Then: + Every target should return only the near pair. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.chrom = b.chrom " + "WHERE DISTANCE(a.interval, b.interval) < 100", + peaks=[("chr1", 100, 200)], + genes=[("chr1", 250, 300), ("chr1", 5000, 5100)], + expected=[(100, 250)], + ) + + def test_distance_no_pair_within_threshold_returns_zero_rows( + self, cross_target_oracle + ): + """Test a column-to-column DISTANCE predicate with no near pair is empty. + + Given: + A peak and a single gene beyond the distance threshold. + When: + The DISTANCE join runs on every target. + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.chrom = b.chrom " + "WHERE DISTANCE(a.interval, b.interval) < 5", + peaks=[("chr1", 100, 200)], + genes=[("chr1", 5000, 5100)], + expected=[], + ) + + +class TestCrossTargetOracleCluster: + """CLUSTER identity across all targets (T2).""" + + def test_cluster_assigns_shared_ids_to_overlapping_runs(self, cross_target_oracle): + """Test CLUSTER assigns identical run ids across targets. + + Given: + Two overlapping intervals and one isolated interval on chr1. + When: + A CLUSTER(interval) projection runs on every target. + Then: + Every target should assign cluster id 1 to the overlapping run and 2 + to the isolated interval, in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end", CLUSTER(interval) AS cid FROM peaks', + peaks=[ + ("chr1", 100, 200), + ("chr1", 150, 300), + ("chr1", 5000, 6000), + ], + expected=[ + ("chr1", 100, 200, 1), + ("chr1", 150, 300, 1), + ("chr1", 5000, 6000, 2), + ], + ) + + def test_cluster_empty_input_returns_zero_rows(self, cross_target_oracle): + """Test CLUSTER over an empty table returns zero rows on every target. + + Given: + An empty peaks table. + When: + The CLUSTER projection runs on every target (DataFusion via the + synthesized empty record batch). + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end", CLUSTER(interval) AS cid FROM peaks', + peaks=[], + expected=[], + ) + + +class TestCrossTargetOracleMerge: + """MERGE identity across all targets (T2).""" + + def test_merge_collapses_overlapping_intervals(self, cross_target_oracle): + """Test MERGE collapses overlapping intervals identically across targets. + + Given: + Two overlapping intervals and one isolated interval on chr1. + When: + A MERGE(interval) query runs on every target. + Then: + Every target should return the merged span and the isolated + interval, in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT MERGE(interval) FROM peaks", + tables=[Table("peaks")], + peaks=[ + ("chr1", 100, 200), + ("chr1", 150, 300), + ("chr1", 5000, 6000), + ], + expected=[("chr1", 100, 300), ("chr1", 5000, 6000)], + ) + + def test_merge_empty_input_returns_zero_rows(self, cross_target_oracle): + """Test MERGE over an empty table returns zero rows on every target. + + Given: + An empty peaks table. + When: + The MERGE query runs on every target (DataFusion via the synthesized + empty record batch). + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT MERGE(interval) FROM peaks", + tables=[Table("peaks")], + peaks=[], + expected=[], + ) + + +class TestCrossTargetOracleDisjoin: + """DISJOIN behaviour: a DuckDB-only case plus the DataFusion duplicate-end gap.""" + + def test_disjoin_splits_overlaps_on_duckdb(self, cross_target_oracle): + """Test DISJOIN splits overlapping intervals at breakpoints on DuckDB. + + Given: + Two overlapping intervals on chr1. + When: + A DISJOIN query runs on the duckdb target only. + Then: + DuckDB should return each original interval paired with the + sub-segments cut at every breakpoint. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end", disjoin_start, disjoin_end FROM DISJOIN(peaks)', + tables=[Table("peaks")], + peaks=[("chr1", 0, 100), ("chr1", 50, 150)], + expected=[ + ("chr1", 0, 100, 0, 50), + ("chr1", 0, 100, 50, 100), + ("chr1", 50, 150, 50, 100), + ("chr1", 50, 150, 100, 150), + ], + targets=("duckdb",), + ) + + @pytest.mark.xfail( + strict=True, + raises=Exception, + reason='DISJOIN CTE * passthrough emits duplicate t."end" columns — #145', + ) + def test_disjoin_full_oracle_xfails_on_datafusion_duplicate_end( + self, cross_target_oracle + ): + """Test the full DISJOIN oracle xfails on DataFusion's duplicate-end names. + + Given: + The same two overlapping intervals. + When: + The oracle runs all three targets — the datafusion target executes + DISJOIN's CTE ``*`` passthrough, which projects ``t."end"`` twice. + Then: + DataFusion should reject the projection for non-unique expression + names (DuckDB tolerates it). This is a strict xfail that promotes + when DISJOIN's passthrough is de-duplicated; an unrelated DataFusion + error escapes and fails loudly. Note: the cause is the duplicate + output-column name, NOT canonicalization or ``* REPLACE`` (no + ``* REPLACE`` is emitted today). + """ + # Arrange / Act + try: + cross_target_oracle( + 'SELECT chrom, start, "end", disjoin_start, disjoin_end ' + "FROM DISJOIN(peaks)", + tables=[Table("peaks")], + peaks=[("chr1", 0, 100), ("chr1", 50, 150)], + expected=[ + ("chr1", 0, 100, 0, 50), + ("chr1", 0, 100, 50, 100), + ("chr1", 50, 150, 50, 100), + ("chr1", 50, 150, 100, 150), + ], + ) + except Exception as exc: # noqa: BLE001 + message = str(exc).lower() + # Assert: narrow to the duplicate-column-name signature so unrelated + # DataFusion failures escape and fail the test loudly. + assert "unique" in message and "name" in message, ( + f"unexpected DataFusion error, not the duplicate-end gap: {exc!r}" + ) + raise + + +class TestCrossTargetOracleDataShapes: + """Data-shape coverage for the oracle (T3).""" + + def test_multi_chrom_join_does_not_cross_chromosomes(self, cross_target_oracle): + """Test a multi-chromosome join keeps overlaps within each chromosome. + + Given: + Peaks on chr1 and chr2 and genes on chr1, chr2, and chr3 where only + same-chromosome pairs overlap by coordinate. + When: + A column-to-column INTERSECTS join runs on every target. + Then: + Every target should return only the same-chromosome overlaps and + never a cross-chromosome pair. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.chrom AS chrom, a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + peaks=[("chr1", 100, 500), ("chr2", 100, 500)], + genes=[("chr1", 300, 400), ("chr2", 300, 400), ("chr3", 300, 400)], + expected=[("chr1", 100, 300), ("chr2", 100, 300)], + ) + + def test_join_path_with_no_overlap_returns_zero_rows(self, cross_target_oracle): + """Test the JOIN path returns zero rows when nothing overlaps. + + Given: + A peak and a gene that do not overlap. + When: + A column-to-column INTERSECTS join runs on every target. + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + peaks=[("chr1", 100, 200)], + genes=[("chr1", 5000, 6000)], + expected=[], + ) + + def test_empty_input_table_returns_zero_rows(self, cross_target_oracle): + """Test an empty input table yields zero rows on every target. + + Given: + An empty peaks table. + When: + A literal INTERSECTS query runs on every target — DataFusion + registers the empty table via the synthesized empty record batch. + Then: + Every target should return zero rows in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end" FROM peaks ' + "WHERE interval INTERSECTS 'chr1:1000-2000'", + peaks=[], + expected=[], + ) + + def test_half_open_touching_intervals_do_not_overlap(self, cross_target_oracle): + """Test half-open adjacent intervals (end == start) do not overlap. + + Given: + A peak ending exactly where a gene begins (half-open adjacency). + When: + A column-to-column INTERSECTS join runs on every target. + Then: + Every target should return zero rows — touching is not overlap under + half-open semantics. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + peaks=[("chr1", 100, 200)], + genes=[("chr1", 200, 300)], + expected=[], + ) + + def test_one_base_pair_overlap_matches(self, cross_target_oracle): + """Test a single-base-pair overlap is counted as an overlap. + + Given: + A peak extending one base past a gene's start. + When: + A column-to-column INTERSECTS join runs on every target. + Then: + Every target should return the single overlapping pair. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + peaks=[("chr1", 100, 201)], + genes=[("chr1", 200, 300)], + expected=[(100, 200)], + ) + + def test_large_interval_spanning_bins_is_not_duplicated(self, cross_target_oracle): + """Test a multi-bin interval yields one pair, not duplicates. + + Given: + A peak spanning many join bins and a single gene inside it. + When: + A column-to-column INTERSECTS join runs on every target. + Then: + Every target should return exactly one pair — the binned join must + dedup the cross-bin candidates, the oracle's multiset compare being + what would catch a duplicate-row regression. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", + peaks=[("chr1", 0, 500000)], + genes=[("chr1", 250000, 250100)], + expected=[(0, 250000)], + ) diff --git a/tests/integration/datafusion/test_oracle_runners.py b/tests/integration/datafusion/test_oracle_runners.py new file mode 100644 index 0000000..423e6a6 --- /dev/null +++ b/tests/integration/datafusion/test_oracle_runners.py @@ -0,0 +1,275 @@ +"""Runner-internals tests for the cross-target oracle (#139, T4). + +These exercise :func:`run_duckdb` / :func:`run_datafusion` directly with trivial +``SELECT ... FROM t`` SQL (no ``transpile``) so any failure localizes to the +runner -- schema construction, type mapping, reserved-word quoting, the +empty-table handling that differs between engines, custom columns, and +cross-runner output-shape parity. +""" + +import pytest + +pytest.importorskip("duckdb") +pytest.importorskip("datafusion") +pytest.importorskip("pyarrow") + +from tests.integration._oracle import arrow_schema # noqa: E402 +from tests.integration._oracle import run_datafusion # noqa: E402 +from tests.integration._oracle import run_duckdb # noqa: E402 + +pytestmark = pytest.mark.integration + +_INTERVAL_COLUMNS = (("chrom", "utf8"), ("start", "int64"), ("end", "int64")) + + +class TestRunDuckDB: + """`run_duckdb` schema, types, quoting, and empty handling.""" + + def test_run_duckdb_returns_normalized_rows(self): + """Test run_duckdb registers a table and returns normalized rows. + + Given: + A two-row interval table and a plain SELECT. + When: + run_duckdb executes the SQL. + Then: + It should return both rows as sorted plain-tuple values. + """ + # Arrange / Act + rows = run_duckdb( + 'SELECT chrom, start, "end" FROM t', + {"t": [("chr2", 5, 6), ("chr1", 1, 2)]}, + _INTERVAL_COLUMNS, + ) + + # Assert + assert rows == [("chr1", 1, 2), ("chr2", 5, 6)] + + def test_run_duckdb_maps_utf8_and_int64_types(self): + """Test run_duckdb maps utf8 to VARCHAR and int64 to BIGINT. + + Given: + A row whose chrom is a string and coordinates are ints. + When: + run_duckdb executes a SELECT. + Then: + The returned values should preserve str and int Python types. + """ + # Arrange / Act + rows = run_duckdb( + "SELECT chrom, start FROM t", {"t": [("chr1", 1, 2)]}, _INTERVAL_COLUMNS + ) + + # Assert + assert isinstance(rows[0][0], str) + assert isinstance(rows[0][1], int) + + def test_run_duckdb_quotes_reserved_end_column(self): + """Test run_duckdb quotes the reserved ``end`` column. + + Given: + A table with the reserved-word ``end`` column. + When: + run_duckdb selects the quoted column. + Then: + It should return the end value without a parse error. + """ + # Arrange / Act + rows = run_duckdb( + 'SELECT "end" FROM t', {"t": [("chr1", 1, 99)]}, _INTERVAL_COLUMNS + ) + + # Assert + assert rows == [(99,)] + + def test_run_duckdb_empty_table_returns_zero_rows(self): + """Test run_duckdb tolerates an empty table via its ``if rows`` guard. + + Given: + An empty interval table. + When: + run_duckdb selects from it. + Then: + It should return zero rows (the guard skips the INSERT). + """ + # Arrange / Act + rows = run_duckdb("SELECT chrom FROM t", {"t": []}, _INTERVAL_COLUMNS) + + # Assert + assert rows == [] + + def test_run_duckdb_custom_columns(self): + """Test run_duckdb registers a custom column schema. + + Given: + A table with a custom ``(name, score)`` schema. + When: + run_duckdb selects from it. + Then: + It should map the custom utf8/int64 columns and return the row. + """ + # Arrange / Act + rows = run_duckdb( + "SELECT name, score FROM t", + {"t": [("gene1", 42)]}, + (("name", "utf8"), ("score", "int64")), + ) + + # Assert + assert rows == [("gene1", 42)] + + +class TestRunDataFusion: + """`run_datafusion` schema, types, quoting, and empty handling.""" + + def test_run_datafusion_returns_normalized_rows(self): + """Test run_datafusion registers a table and returns normalized rows. + + Given: + A two-row interval table and a plain SELECT. + When: + run_datafusion executes the SQL. + Then: + It should return both rows as sorted plain-tuple values. + """ + # Arrange / Act + rows = run_datafusion( + 'SELECT chrom, start, "end" FROM t', + {"t": [("chr2", 5, 6), ("chr1", 1, 2)]}, + _INTERVAL_COLUMNS, + ) + + # Assert + assert rows == [("chr1", 1, 2), ("chr2", 5, 6)] + + def test_run_datafusion_maps_utf8_and_int64_types(self): + """Test run_datafusion maps utf8 to pyarrow utf8 and int64 to int64. + + Given: + A row whose chrom is a string and coordinates are ints. + When: + run_datafusion executes a SELECT. + Then: + The returned values should be plain str and int after coercion. + """ + # Arrange / Act + rows = run_datafusion( + "SELECT chrom, start FROM t", {"t": [("chr1", 1, 2)]}, _INTERVAL_COLUMNS + ) + + # Assert + assert isinstance(rows[0][0], str) + assert isinstance(rows[0][1], int) + + def test_run_datafusion_quotes_reserved_end_column(self): + """Test run_datafusion quotes the reserved ``end`` column. + + Given: + A table with the reserved-word ``end`` column. + When: + run_datafusion selects the quoted column. + Then: + It should return the end value without a parse error. + """ + # Arrange / Act + rows = run_datafusion( + 'SELECT "end" FROM t', {"t": [("chr1", 1, 99)]}, _INTERVAL_COLUMNS + ) + + # Assert + assert rows == [(99,)] + + def test_run_datafusion_empty_table_returns_zero_rows(self): + """Test run_datafusion handles an empty table via a synthesized batch. + + Given: + An empty interval table (which would panic raw + ``register_record_batches``). + When: + run_datafusion selects from it. + Then: + It should return zero rows, proving the loader synthesizes an empty + record batch where DuckDB instead skips the INSERT. + """ + # Arrange / Act + rows = run_datafusion("SELECT chrom FROM t", {"t": []}, _INTERVAL_COLUMNS) + + # Assert + assert rows == [] + + def test_run_datafusion_custom_columns(self): + """Test run_datafusion registers a custom column schema. + + Given: + A table with a custom ``(name, score)`` schema. + When: + run_datafusion selects from it. + Then: + It should map the custom utf8/int64 columns and return the row. + """ + # Arrange / Act + rows = run_datafusion( + "SELECT name, score FROM t", + {"t": [("gene1", 42)]}, + (("name", "utf8"), ("score", "int64")), + ) + + # Assert + assert rows == [("gene1", 42)] + + +class TestRawRegisterRecordBatchesEmpty: + """Pin the raw DataFusion empty-table panic the runner guards against.""" + + def test_raw_register_record_batches_panics_on_empty(self): + """Test raw register_record_batches panics on a truly-empty partition. + + Given: + An empty pyarrow table whose ``to_batches()`` yields no batches. + When: + ``SessionContext.register_record_batches`` is called with it + directly (NOT via the oracle helper). + Then: + DataFusion should raise/panic on the empty partition, documenting + why :func:`run_datafusion` synthesizes an empty batch instead. + """ + # Arrange + import pyarrow as pa + from datafusion import SessionContext + + schema = arrow_schema(_INTERVAL_COLUMNS) + ctx = SessionContext() + empty_batches = pa.table( + {"chrom": [], "start": [], "end": []}, schema=schema + ).to_batches() + assert empty_batches == [] # no batches is the panic trigger + + # Act / Assert + with pytest.raises(BaseException): + ctx.register_record_batches("t", [empty_batches]) + + +class TestCrossRunnerParity: + """Output-shape parity between the two runners on trivial SQL.""" + + def test_runners_agree_on_shape_and_values(self): + """Test both runners produce identical normalized output for one query. + + Given: + The same table data and trivial SELECT for both engines. + When: + run_duckdb and run_datafusion each execute it. + Then: + Their normalized results should be identical -- the parity the + oracle relies on for its differential comparison. + """ + # Arrange + data = {"t": [("chr1", 1, 2), ("chr2", 3, 4)]} + sql = 'SELECT chrom, start, "end" FROM t' + + # Act + ddb = run_duckdb(sql, data, _INTERVAL_COLUMNS) + df = run_datafusion(sql, data, _INTERVAL_COLUMNS) + + # Assert + assert ddb == df diff --git a/tests/integration/test_oracle_lane.py b/tests/integration/test_oracle_lane.py new file mode 100644 index 0000000..d50c276 --- /dev/null +++ b/tests/integration/test_oracle_lane.py @@ -0,0 +1,208 @@ +"""Capability-truthing and lane/CI guards for the cross-target oracle (#139, T5). + +These pin the invariants the oracle's routing and xfails depend on: the +DataFusion target's declared capabilities (no LATERAL, no ``* REPLACE``), that +the engine-free oracle internals are importable and reusable from a non- +DataFusion lane, the skip-path behaviour when an engine is absent, and a +version-floor guard for the optional engines so a too-old DuckDB / DataFusion +fails with a clear message rather than a cryptic runtime error. +""" + +import importlib.util + +import pytest + +from giql.targets import DataFusionTarget +from giql.targets import DuckDBTarget +from tests.integration._oracle import assert_cross_target +from tests.integration._oracle import normalize +from tests.integration._oracle import resolve_routing + + +class TestDataFusionCapabilityTruthing: + """The DataFusion capability flags the oracle's xfails rely on.""" + + def test_datafusion_does_not_support_lateral(self): + """Test DataFusionTarget declares no LATERAL support. + + Given: + A DataFusionTarget instance. + When: + Its capabilities are inspected. + Then: + supports_lateral should be False, matching the NEAREST xfail (#142). + """ + # Arrange / Act / Assert + assert DataFusionTarget().capabilities.supports_lateral is False + + def test_datafusion_does_not_support_star_replace(self): + """Test DataFusionTarget declares no ``* REPLACE`` support. + + Given: + A DataFusionTarget instance. + When: + Its capabilities are inspected. + Then: + supports_star_replace should be False (DataFusion has EXCEPT/EXCLUDE + only). + """ + # Arrange / Act / Assert + assert DataFusionTarget().capabilities.supports_star_replace is False + + def test_duckdb_supports_star_replace(self): + """Test DuckDBTarget declares ``* REPLACE`` support as a contrast. + + Given: + A DuckDBTarget instance. + When: + Its capabilities are inspected. + Then: + supports_star_replace should be True, confirming the asymmetry the + DISJOIN note describes is about duplicate output names, not a + ``* REPLACE`` capability gap. + """ + # Arrange / Act / Assert + assert DuckDBTarget().capabilities.supports_star_replace is True + + +class TestSharedOracleReuse: + """The engine-free internals are importable from any lane.""" + + def test_oracle_internals_reusable_without_engines(self): + """Test the comparison core works imported from a non-DataFusion lane. + + Given: + Two agreeing normalized results assembled with no engine present. + When: + assert_cross_target compares them against a matching expectation. + Then: + It should pass, proving the shared module is reusable outside the + DataFusion lane (e.g. for the bedtools / coordinate_space lanes). + """ + # Arrange + rows = normalize([("chr1", 1, 2)]) + + # Act / Assert + assert_cross_target({"a": rows, "b": rows}, normalize([("chr1", 1, 2)])) + + def test_routing_resolves_without_engines(self): + """Test resolve_routing is a pure function needing no engine import. + + Given: + The default targets. + When: + resolve_routing resolves them with no engine installed or imported. + Then: + It should return the full routing map, confirming routing is a pure + decision (Finding 7). + """ + # Arrange / Act + routing = resolve_routing(("generic", "datafusion", "duckdb")) + + # Assert + assert set(routing) == {"generic", "datafusion", "duckdb"} + + +class TestEngineSkipPaths: + """Skip / availability behaviour for the optional engines.""" + + def test_duckdb_available_or_skipped(self): + """Test the DuckDB skip-path: present engine imports, absent one skips. + + Given: + The optional DuckDB dependency, which may or may not be installed. + When: + The lane probes for it via importorskip. + Then: + It should import when present (and otherwise skip cleanly), mirroring + the lane's module-level guard. + """ + # Arrange / Act + duckdb = pytest.importorskip("duckdb") + + # Assert + assert hasattr(duckdb, "connect") + + def test_datafusion_available_or_skipped(self): + """Test the DataFusion skip-path: present engine imports, absent skips. + + Given: + The optional DataFusion and pyarrow dependencies. + When: + The lane probes for them via importorskip. + Then: + They should import when present (and otherwise skip cleanly). + """ + # Arrange / Act + datafusion = pytest.importorskip("datafusion") + pytest.importorskip("pyarrow") + + # Assert + assert hasattr(datafusion, "SessionContext") + + +class TestEngineVersionFloor: + """Version-floor guards for the optional engines.""" + + def test_duckdb_meets_version_floor(self): + """Test the installed DuckDB meets the lane's minimum version. + + Given: + The optional DuckDB dependency. + When: + Its version tuple is compared to the floor the IEJoin SQL needs. + Then: + It should be at least the supported floor, failing loudly otherwise. + """ + # Arrange + duckdb = pytest.importorskip("duckdb") + floor = (0, 9) + + # Act + version = tuple(int(p) for p in duckdb.__version__.split(".")[:2]) + + # Assert + assert version >= floor, f"DuckDB {duckdb.__version__} below floor {floor}" + + def test_datafusion_meets_version_floor(self): + """Test the installed DataFusion meets the lane's minimum version. + + Given: + The optional DataFusion dependency. + When: + Its version tuple is compared to the floor the lane was validated on. + Then: + It should be at least the supported floor, failing loudly otherwise. + """ + # Arrange + datafusion = pytest.importorskip("datafusion") + floor = (40, 0) + + # Act + version = tuple(int(p) for p in datafusion.__version__.split(".")[:2]) + + # Assert + assert version >= floor, ( + f"DataFusion {datafusion.__version__} below floor {floor}" + ) + + +class TestLaneCollection: + """A smoke check that the lane and oracle modules collect.""" + + def test_oracle_modules_are_importable(self): + """Test the oracle's importable modules resolve by spec. + + Given: + The non-test ``_oracle`` module and the conftest-backed fixture. + When: + Their import specs are looked up. + Then: + The ``_oracle`` module should be findable, anchoring the lane's + shared-helper layout. + """ + # Arrange / Act + spec = importlib.util.find_spec("tests.integration._oracle") + + # Assert + assert spec is not None From b10dec6d0a900a78ad4538f6a812825ed0f06c66 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 14 Jun 2026 20:04:01 -0400 Subject: [PATCH 079/142] test: Fix determinism self-test method name typo --- tests/integration/test_oracle_internals.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_oracle_internals.py b/tests/integration/test_oracle_internals.py index ceb4398..e20de78 100644 --- a/tests/integration/test_oracle_internals.py +++ b/tests/integration/test_oracle_internals.py @@ -129,7 +129,7 @@ def test_normalize_preserves_multiplicity(self): # Assert assert result == [("chr1", 1, 2), ("chr1", 1, 2)] - def test_normalize_is_determinor_under_shuffle(self): + def test_normalize_is_deterministic_under_shuffle(self): """Test normalize yields an identical result under row shuffling. Given: From 0c6b3d76ce4153867598797a908f31211a2f37c4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:38:07 -0400 Subject: [PATCH 080/142] fix: Harden ExpandOperators dispatch and registry surface Replace pending nodes deepest-first so an ancestor expansion can no longer detach a still-pending descendant, whose later replace() would silently no-op and leave the inner operator unexpanded. Skip any node already detached as a second line of defence. Raise ResolutionError when an operator reaches the pass without valid resolution metadata rather than minting a None-resolution context, and raise TypeError when an expander returns a non-Expression. Require an OperatorExpander's expand attribute to be callable before trusting the runtime-checkable protocol branch. Guard register() class instantiation with a clear message when a custom Target subclass has required fields. Add __len__/__bool__ to ExpanderRegistry for public emptiness checks. Closes #154 --- src/giql/expander.py | 86 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 9 deletions(-) diff --git a/src/giql/expander.py b/src/giql/expander.py index 34f742d..f93132a 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -57,6 +57,7 @@ from giql.resolver import META_KEY from giql.resolver import OperatorResolution +from giql.resolver import ResolutionError from giql.table import Tables from giql.targets import GenericTarget from giql.targets import Target @@ -77,11 +78,6 @@ #: mirroring the ``__giql_canon_`` / ``__giql_dj_`` reserved prefixes. EXPAND_ALIAS_PREFIX = "__giql_x_" -#: The GIQL operator expression classes the pass inspects. Membership alone does -#: not opt an operator in: the per-class ``GIQL_EXPAND`` flag plus a registered -#: expander do (see module docstring). Imported lazily inside the pass to avoid a -#: module-level import cycle with :mod:`giql.expressions`. - class ExpansionContext: """Everything an :class:`OperatorExpander` needs to expand one operator. @@ -165,7 +161,10 @@ def expand(self, node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: def _as_callable(expander: OperatorExpander | ExpanderFn) -> ExpanderFn: """Normalize an expander to a plain ``(node, ctx) -> Expression`` callable.""" - if isinstance(expander, OperatorExpander): + # ``@runtime_checkable`` only checks an ``expand`` attribute *exists*, so an + # object with a non-callable ``expand`` would pass the isinstance check; guard + # that the attribute is actually callable before trusting the protocol branch. + if isinstance(expander, OperatorExpander) and callable(expander.expand): return expander.expand if callable(expander): return expander @@ -265,6 +264,20 @@ def __contains__(self, key: tuple[Target, type]) -> bool: """ return key in self._expanders + def __len__(self) -> int: + """The number of registered ``(target, operator)`` entries. + + Part of the public introspection surface (alongside ``in``): emptiness is + observable without reaching into private state, so a teardown fixture or + leak guard can assert the registry is clear via ``len(registry)`` or + ``bool(registry)``. + """ + return len(self._expanders) + + def __bool__(self) -> bool: + """Whether any expander is registered (``False`` when empty).""" + return bool(self._expanders) + #: The process-wide registry the :func:`register` decorator writes to and the #: :class:`ExpandOperators` pass reads from. Empty as of this issue, so the pass @@ -299,7 +312,19 @@ def expand_disjoin(node, ctx): ... A decorator that registers its argument and returns it unchanged, so the decorated object stays usable on its own. """ - target_instance = target() if isinstance(target, type) else target + if isinstance(target, type): + try: + target_instance: Target = target() + except TypeError as exc: + raise TypeError( + f"register() could not instantiate target class {target.__name__!r} " + "with no arguments: a custom Target subclass with required fields " + "cannot be defaulted. Pass an instance (e.g. " + f"@register({target.__name__}(...), ...)) or give its fields " + "defaults so the class form can construct it." + ) from exc + else: + target_instance = target def decorator( expander: OperatorExpander | ExpanderFn, @@ -310,8 +335,13 @@ def decorator( return decorator +# The GIQL operator expression classes the pass inspects. Membership alone does +# not opt an operator in: the per-class ``GIQL_EXPAND`` flag plus a registered +# expander do (see module docstring). Imported lazily as a precaution; there is no +# real cycle (``canonicalizer.py`` imports the same classes eagerly at module +# level), so a later step may hoist these to module scope to match that sibling. def _giql_operators() -> tuple[type, ...]: - """Return the GIQL operator classes, imported lazily to avoid a cycle.""" + """Return the GIQL operator classes, imported lazily as a precaution.""" from giql.expressions import Contains from giql.expressions import GIQLCluster from giql.expressions import GIQLDisjoin @@ -389,18 +419,56 @@ class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for continue pending.append((node, fn)) + # Replace deepest-first: replacing an ancestor node detaches any collected + # descendant, whose later ``node.replace`` would be a silent no-op (the + # detached node's parent is gone), so the inner operator would never expand. + # Mutating leaves first keeps every still-collected ancestor attached. A + # detached node is skipped defensively as a second line of defence (#154). + pending.sort(key=lambda item: _node_depth(item[0]), reverse=True) + for node, fn in pending: + if node is not expression and node.parent is None: + # An ancestor expansion already detached this node from the tree; + # replacing it would be a no-op, so skip it. + continue resolution = node.meta.get(META_KEY) if not isinstance(resolution, OperatorResolution): - resolution = None + # Pass 1 guarantees every operator node carries its resolution; a + # missing or malformed one is an internal invariant violation, not a + # user error, so fail loudly rather than minting a None-resolution + # context the expander would then dereference blindly. + raise ResolutionError( + f"{type(node).__name__} reached the ExpandOperators pass without " + "valid resolution metadata; pass 1 (resolve_operator_refs) must " + "run first and annotate every operator node." + ) ctx = ExpansionContext(node, resolution, target, tables, alias_seq) replacement = fn(node, ctx) + if not isinstance(replacement, exp.Expression): + raise TypeError( + f"expander {fn!r} for {type(node).__name__} returned " + f"{type(replacement).__name__}, not exp.Expression" + ) if replacement is not node: node.replace(replacement) return expression +def _node_depth(node: exp.Expression) -> int: + """The number of ancestors above *node* (root is depth 0). + + Used to order the collect-then-replace loop deepest-first so replacing an + ancestor never detaches a still-pending descendant before its own replace. + """ + depth = 0 + parent = node.parent + while parent is not None: + depth += 1 + parent = parent.parent + return depth + + class ExpandOperators: """Callable wrapper for :func:`expand_operators`, parallel to the transformers. From 3735154d3790ce92c9b50b4663e39ac7672665a9 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:38:08 -0400 Subject: [PATCH 081/142] docs: Note CLUSTER/MERGE GIQL_EXPAND is inert today Their transformers rewrite the nodes before the ExpandOperators pass runs, so the flag is forward-looking for #144, not a live opt-in. --- src/giql/expressions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 4ba384a..ce939dc 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -232,6 +232,10 @@ class GIQLCluster(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } + # Inert today: the CLUSTER/MERGE transformers rewrite these nodes before the + # ExpandOperators pass runs, so the pass never sees a GIQLCluster to dispatch + # and this flag is not a live opt-in. It is forward-looking for #144, which + # migrates these operators onto the expander registry. GIQL_EXPAND = _EXPAND @classmethod @@ -275,6 +279,10 @@ class GIQLMerge(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } + # Inert today: the CLUSTER/MERGE transformers rewrite these nodes before the + # ExpandOperators pass runs, so the pass never sees a GIQLMerge to dispatch + # and this flag is not a live opt-in. It is forward-looking for #144, which + # migrates these operators onto the expander registry. GIQL_EXPAND = _EXPAND @classmethod From f6bc9beef70f519050ddf19b6d54a7e837f640a0 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:48:23 -0400 Subject: [PATCH 082/142] fix: Harden oracle normalize and guard empty targets Make the normalize sort key total via type-name tiebreak so a mixed-type column surfaces as a clean multiset inequality, coerce integral floats back to int for DataFusion's NULL-driven int64->float64 promotion, normalize resolve_routing targets to a tuple so a generator is not double-consumed, and assert run_targets is non-empty so an empty results dict can no longer pass vacuously. --- tests/integration/_oracle.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/integration/_oracle.py b/tests/integration/_oracle.py index f0bd377..d3b9a5d 100644 --- a/tests/integration/_oracle.py +++ b/tests/integration/_oracle.py @@ -66,6 +66,11 @@ def scalar(value): return None if isinstance(value, float) and math.isnan(value): return None + # DataFusion promotes int64 -> float64 on NULL-bearing columns (1 -> 1.0). + # Coerce an integral float back to int so a NULL-bearing column compares + # type-stably against DuckDB's plain ints in the multiset. + if isinstance(value, float) and value.is_integer(): + return int(value) return value @@ -77,7 +82,11 @@ def normalize(rows) -> list[tuple]: scalars so DuckDB and DataFusion (pandas) rows compare equal. """ out = [tuple(scalar(v) for v in row) for row in rows] - return sorted(out, key=lambda r: tuple((v is None, v) for v in r)) + # Include ``type(v).__name__`` in the key so a column carrying mixed scalar + # types sorts on a total order instead of raising ``TypeError`` on an + # ``int`` < ``str`` comparison. A genuine type divergence then surfaces as a + # clean multiset inequality rather than a crash. + return sorted(out, key=lambda r: tuple((v is None, type(v).__name__, v) for v in r)) def resolve_routing(targets, engines=None) -> dict[str, tuple[str, str | None]]: @@ -108,6 +117,10 @@ def resolve_routing(targets, engines=None) -> dict[str, tuple[str, str | None]]: If a target name is not a known target, or an override routes to an unknown engine. """ + # Normalize to a tuple so a one-shot generator is not exhausted here and + # then silently empty when the fixture iterates ``targets`` again. + targets = tuple(targets) + engine_for = dict(TARGET_ENGINE) if engines: for target, engine in engines.items(): @@ -160,6 +173,10 @@ def assert_cross_target(results, expected, sql_by_target=None) -> None: sql_by_target = sql_by_target or {} run_targets = list(results) + # Guard against a vacuous pass: an empty ``results`` would skip both phases + # and never check ``expected`` at all. + assert run_targets, "oracle invoked with no targets" + # Phase 1: engines must agree with each other (the differential oracle). if len(run_targets) > 1: reference = run_targets[0] From b9755a3a971a1b7d7f3ec768562bd701b5a42071 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:48:23 -0400 Subject: [PATCH 083/142] fix: Reject custom oracle columns without explicit tables Raise ValueError when columns diverge from the default chrom/start/end schema but no tables= list is given, since the default table mapping would silently mis-register the custom column names. --- tests/integration/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 80bcdd7..b0a96e1 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -119,6 +119,13 @@ def _oracle( **table_data, ) -> dict[str, list[tuple]]: if tables is None: + if columns != DEFAULT_COLUMNS: + raise ValueError( + "Custom 'columns' require an explicit 'tables=' list: the " + "default table mapping registers chrom/start/end, which " + "would silently mis-register the custom column names. Pass " + "tables=[Table(name, ...)] for each table." + ) tables = [_default_table(name, columns) for name in table_data] routing = resolve_routing(targets, engines) From 9f81c9011023699fb5f8615dbb91256382b2176c Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:48:31 -0400 Subject: [PATCH 084/142] test: Replace inert strict-xfail with loud pytest.raises gaps Convert the NEAREST (#142) and DISJOIN (#153) DataFusion gap tests from a strict-xfail plus in-body narrowing (which swallowed the narrowing AssertionError and recorded XFAIL on unrelated errors) to plain pytest.raises(match=...) tests that fail loudly on a reworded/unrelated error and trip DID NOT RAISE when the gap closes. Re-point DISJOIN to the real cause: unaliased duplicate end columns in the __giql_dj_cuts UNION branches. Widen the dedup test so the two intervals share many bins, forcing duplicate candidates a stripped DISTINCT would surface, add DISTANCE upstream-gap and overlap branch cases, and temper the structurally-different SQL framing for the non-join operators. --- .../datafusion/test_cross_target_oracle.py | 151 +++++++++++------- 1 file changed, 94 insertions(+), 57 deletions(-) diff --git a/tests/integration/datafusion/test_cross_target_oracle.py b/tests/integration/datafusion/test_cross_target_oracle.py index 5cffd69..6b717a1 100644 --- a/tests/integration/datafusion/test_cross_target_oracle.py +++ b/tests/integration/datafusion/test_cross_target_oracle.py @@ -7,11 +7,20 @@ expander registry yet (epic #137), so this lane locks in the verification path every later migration (#140-#144) will consume. +For the non-join operators (DISTANCE, CONTAINS, WITHIN, ANY/ALL, CLUSTER, +MERGE) the generic and datafusion targets emit byte-identical SQL and both run +on the DataFusion engine, so the load-bearing comparison there is +DataFusion-vs-DuckDB. Only the column-to-column INTERSECTS join produces +genuinely divergent SQL across targets (the DuckDB IEJoin vs. the binned +equi-join). + NEAREST's expansion uses a correlated ``LATERAL`` subquery, which DataFusion has no physical plan for today; its generic-vs-duckdb equivalence case runs both on -DuckDB, and the full three-target oracle is pinned as a strict xfail (#142) that -auto-promotes when DataFusion gains correlated LATERAL — the one documented -cross-target gap. +DuckDB, and the full three-target oracle is pinned by a +``pytest.raises(match="OuterReferenceColumn")`` test (#142) that fails loudly on +an unrelated error and trips "DID NOT RAISE" — forcing conversion to a real +identity test — when DataFusion gains correlated LATERAL. DISJOIN has an +analogous pending-#153 gap (duplicate ``end`` output names). """ import pytest @@ -195,13 +204,8 @@ def test_standalone_nearest_k1_agrees_generic_vs_duckdb_on_duckdb( engines={"generic": "duckdb"}, ) - @pytest.mark.xfail( - strict=True, - raises=Exception, - reason="DataFusion lacks correlated LATERAL (OuterReferenceColumn) — #142", - ) - def test_nearest_full_oracle_xfails_on_datafusion_lateral(self, cross_target_oracle): - """Test the full NEAREST oracle xfails on DataFusion's missing LATERAL. + def test_nearest_on_datafusion_unsupported_pending_142(self, cross_target_oracle): + """Test the full NEAREST oracle raises DataFusion's missing-LATERAL error. Given: The single-row NEAREST query and a candidate gene on chr1. @@ -210,13 +214,14 @@ def test_nearest_full_oracle_xfails_on_datafusion_lateral(self, cross_target_ora the correlated LATERAL on DataFusion, which has no physical plan. Then: DataFusion should raise its ``OuterReferenceColumn`` "not - implemented" error, pinning the gap as a strict xfail that - auto-promotes (XPASS -> fail) when #142 lands. An UNRELATED - DataFusion error must still fail loudly, so the match is narrowed - to the LATERAL signature before re-raising. + implemented" error. This pins the known #142 gap: the ``match`` + narrows to the LATERAL signature so an unrelated/reworded DataFusion + error fails loudly, and a closed gap (no exception) trips pytest's + "DID NOT RAISE", forcing this to be converted into a real + cross-target identity test when DataFusion gains correlated LATERAL. """ - # Arrange / Act - try: + # Arrange / Act / Assert + with pytest.raises(Exception, match="OuterReferenceColumn"): cross_target_oracle( "SELECT a.chrom, a.start AS a_start, b.start AS b_start " "FROM peaks a " @@ -225,14 +230,6 @@ def test_nearest_full_oracle_xfails_on_datafusion_lateral(self, cross_target_ora genes=[("chr1", 280, 290)], expected=[("chr1", 200, 280)], ) - except Exception as exc: # noqa: BLE001 - message = str(exc).lower() - # Assert: narrow the xfail to the documented LATERAL gap so any - # unrelated DataFusion failure escapes and fails the test loudly. - assert "outerreferencecolumn" in message or "not implemented" in message, ( - f"unexpected DataFusion error, not the LATERAL gap: {exc!r}" - ) - raise class TestCrossTargetOracleIntersectsAnyAll: @@ -344,6 +341,49 @@ def test_distance_column_to_column_filters_near_pairs(self, cross_target_oracle) expected=[(100, 250)], ) + def test_distance_upstream_gap_b_precedes_a(self, cross_target_oracle): + """Test DISTANCE measures the upstream gap when B precedes A. + + Given: + A peak and a gene that ends before the peak begins on the same + chromosome, exercising the ``ELSE`` (upstream-gap) CASE branch. + When: + A DISTANCE(a, b) < threshold join runs on every target. + Then: + Every target should return the pair, the gap being measured from B's + end to A's start. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.chrom = b.chrom " + "WHERE DISTANCE(a.interval, b.interval) < 100", + peaks=[("chr1", 250, 300)], + genes=[("chr1", 100, 200)], + expected=[(250, 100)], + ) + + def test_distance_overlapping_pair_is_zero(self, cross_target_oracle): + """Test DISTANCE is zero for overlapping intervals. + + Given: + A peak and a gene that overlap on the same chromosome, exercising the + overlap (``THEN 0``) CASE branch. + When: + A DISTANCE(a, b) < threshold join runs on every target. + Then: + Every target should return the pair, the overlap yielding distance 0. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a JOIN genes b ON a.chrom = b.chrom " + "WHERE DISTANCE(a.interval, b.interval) < 100", + peaks=[("chr1", 100, 300)], + genes=[("chr1", 200, 400)], + expected=[(100, 200)], + ) + def test_distance_no_pair_within_threshold_returns_zero_rows( self, cross_target_oracle ): @@ -462,7 +502,12 @@ def test_merge_empty_input_returns_zero_rows(self, cross_target_oracle): class TestCrossTargetOracleDisjoin: - """DISJOIN behaviour: a DuckDB-only case plus the DataFusion duplicate-end gap.""" + """DISJOIN: a DuckDB-only case plus the DataFusion duplicate-``end`` gap (#153). + + The DataFusion gap is the unaliased duplicate ``t."end"`` columns in the + ``__giql_dj_cuts`` CTE UNION branches, which DataFusion rejects as non-unique + projection names (DuckDB tolerates them). + """ def test_disjoin_splits_overlaps_on_duckdb(self, cross_target_oracle): """Test DISJOIN splits overlapping intervals at breakpoints on DuckDB. @@ -489,31 +534,28 @@ def test_disjoin_splits_overlaps_on_duckdb(self, cross_target_oracle): targets=("duckdb",), ) - @pytest.mark.xfail( - strict=True, - raises=Exception, - reason='DISJOIN CTE * passthrough emits duplicate t."end" columns — #145', - ) - def test_disjoin_full_oracle_xfails_on_datafusion_duplicate_end( - self, cross_target_oracle - ): - """Test the full DISJOIN oracle xfails on DataFusion's duplicate-end names. + def test_disjoin_on_datafusion_unsupported_pending_153(self, cross_target_oracle): + """Test the full DISJOIN oracle raises DataFusion's duplicate-name error. Given: The same two overlapping intervals. When: The oracle runs all three targets — the datafusion target executes - DISJOIN's CTE ``*`` passthrough, which projects ``t."end"`` twice. + DISJOIN's ``__giql_dj_cuts`` CTE, whose UNION branches project the + unaliased ``t."end"`` column twice. Then: DataFusion should reject the projection for non-unique expression - names (DuckDB tolerates it). This is a strict xfail that promotes - when DISJOIN's passthrough is de-duplicated; an unrelated DataFusion - error escapes and fails loudly. Note: the cause is the duplicate - output-column name, NOT canonicalization or ``* REPLACE`` (no - ``* REPLACE`` is emitted today). + names (DuckDB tolerates it). This pins the known #153 gap: the + ``match`` narrows to the duplicate-output-name signature so an + unrelated/reworded DataFusion error fails loudly, and a closed gap + (no exception) trips pytest's "DID NOT RAISE", forcing this to be + converted into a real cross-target identity test when the duplicate + ``end`` columns in the ``__giql_dj_cuts`` UNION branches are aliased. """ - # Arrange / Act - try: + # Arrange / Act / Assert + with pytest.raises( + Exception, match="Projections require unique expression names" + ): cross_target_oracle( 'SELECT chrom, start, "end", disjoin_start, disjoin_end ' "FROM DISJOIN(peaks)", @@ -526,14 +568,6 @@ def test_disjoin_full_oracle_xfails_on_datafusion_duplicate_end( ("chr1", 50, 150, 100, 150), ], ) - except Exception as exc: # noqa: BLE001 - message = str(exc).lower() - # Assert: narrow to the duplicate-column-name signature so unrelated - # DataFusion failures escape and fail the test loudly. - assert "unique" in message and "name" in message, ( - f"unexpected DataFusion error, not the duplicate-end gap: {exc!r}" - ) - raise class TestCrossTargetOracleDataShapes: @@ -638,22 +672,25 @@ def test_one_base_pair_overlap_matches(self, cross_target_oracle): ) def test_large_interval_spanning_bins_is_not_duplicated(self, cross_target_oracle): - """Test a multi-bin interval yields one pair, not duplicates. + """Test two intervals sharing many bins yield one pair, not duplicates. Given: - A peak spanning many join bins and a single gene inside it. + A peak (0-500000) and a gene (5000-495000) that co-occupy dozens of + shared join bins (the bin width is 10000), so the binned candidate + join produces the same pair once per shared bin. When: A column-to-column INTERSECTS join runs on every target. Then: - Every target should return exactly one pair — the binned join must - dedup the cross-bin candidates, the oracle's multiset compare being - what would catch a duplicate-row regression. + Every target should return exactly one pair — the binned join's + ``SELECT DISTINCT`` must collapse the duplicate cross-bin candidates, + and the oracle's multiset compare is what would catch a stripped + DISTINCT as a duplicate-row regression. """ # Arrange / Act / Assert cross_target_oracle( "SELECT a.start AS a_start, b.start AS b_start " "FROM peaks a JOIN genes b ON a.interval INTERSECTS b.interval", peaks=[("chr1", 0, 500000)], - genes=[("chr1", 250000, 250100)], - expected=[(0, 250000)], + genes=[("chr1", 5000, 495000)], + expected=[(0, 5000)], ) From 5191bd0c267b407b5363bc355bd4bc874aeab2ba Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:48:37 -0400 Subject: [PATCH 085/142] test: Add 3-target, property, and split coercion oracle tests Add a 3-target positive control and last-target-diverges negative to mirror the production three-target spread, split the dual-behavior coercion test into single-behavior pass/raise tests, add Hypothesis property tests for normalize order-invariance and idempotence and resolve_routing totality, and mark the module integration so pytest -m integration collects it. --- tests/integration/test_oracle_internals.py | 156 +++++++++++++++++++-- 1 file changed, 146 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_oracle_internals.py b/tests/integration/test_oracle_internals.py index e20de78..af8155c 100644 --- a/tests/integration/test_oracle_internals.py +++ b/tests/integration/test_oracle_internals.py @@ -11,12 +11,16 @@ import math import pytest +from hypothesis import given +from hypothesis import strategies as st from tests.integration._oracle import assert_cross_target from tests.integration._oracle import normalize from tests.integration._oracle import resolve_routing from tests.integration._oracle import scalar +pytestmark = pytest.mark.integration + class _FakeNumpyScalar: """A scalar that only reveals its value (and NaN-ness) after ``.item()``.""" @@ -319,29 +323,43 @@ def test_raises_on_none_versus_value_difference(self): with pytest.raises(AssertionError): assert_cross_target(results, normalize([("chr1", 5, 2)])) - def test_coercion_normalizes_type_but_preserves_value_diff(self): - """Test type coercion does not mask a genuine value difference. + def test_coercion_passes_when_coerced_value_matches_expected(self): + """Test type coercion lets a coerced scalar compare equal to a plain int. Given: - Targets whose values are numpy-like scalars equal to expected ints - in one case and differing in another. + Targets whose values are numpy-like scalars equal to the expected + plain ints once coerced. When: assert_cross_target() compares the coerced multisets. Then: - The equal case passes and the differing case raises — coercion - normalizes type without hiding a real difference. + It should pass — coercion normalizes type so the values match. """ # Arrange - equal_rows = normalize([("chr1", _FakeNumpyScalar(1), _FakeNumpyScalar(2))]) - diff_rows = normalize([("chr1", _FakeNumpyScalar(1), _FakeNumpyScalar(2))]) + rows = normalize([("chr1", _FakeNumpyScalar(1), _FakeNumpyScalar(2))]) # Act / Assert assert_cross_target( - {"generic": equal_rows, "duckdb": equal_rows}, normalize([("chr1", 1, 2)]) + {"generic": rows, "duckdb": rows}, normalize([("chr1", 1, 2)]) ) + + def test_coercion_preserves_genuine_value_difference(self): + """Test type coercion does not mask a genuine value difference. + + Given: + Targets whose coerced values genuinely differ from expected. + When: + assert_cross_target() compares the coerced multisets. + Then: + It should raise — coercion normalizes type without hiding a real + difference. + """ + # Arrange + rows = normalize([("chr1", _FakeNumpyScalar(1), _FakeNumpyScalar(2))]) + + # Act / Assert with pytest.raises(AssertionError): assert_cross_target( - {"generic": diff_rows, "duckdb": diff_rows}, + {"generic": rows, "duckdb": rows}, normalize([("chr1", 1, 99)]), ) @@ -371,6 +389,49 @@ def test_raises_with_engines_disagree_message(self): assert "engines disagree" in message assert "generic" in message and "duckdb" in message + def test_three_targets_all_agree_passes(self): + """Test three agreeing targets pass, mirroring the production spread. + + Given: + Three targets (generic, datafusion, duckdb) returning identical rows + and a matching expectation — the real three-target run shape. + When: + assert_cross_target() runs. + Then: + It should not raise. + """ + # Arrange + rows = normalize([("chr1", 1, 2)]) + results = {"generic": rows, "datafusion": rows, "duckdb": rows} + + # Act / Assert + assert_cross_target(results, normalize([("chr1", 1, 2)])) + + def test_three_targets_last_diverges_raises(self): + """Test a divergence in only the LAST of three targets still raises. + + Given: + Three targets where the first two agree but the last (duckdb) + returns a different row — the differential phase compares each target + against the reference, so a trailing divergence must not slip past. + When: + assert_cross_target() runs. + Then: + It should raise the engines-disagree error naming the divergent + target. + """ + # Arrange + rows = normalize([("chr1", 1, 2)]) + results = { + "generic": rows, + "datafusion": rows, + "duckdb": normalize([("chr1", 9, 9)]), + } + + # Act / Assert + with pytest.raises(AssertionError, match="engines disagree"): + assert_cross_target(results, rows) + def test_disagreement_checked_before_expectation(self): """Test the differential phase fires even when one target matches expected. @@ -475,3 +536,78 @@ def test_unknown_target_in_override_raises_value_error(self): # Arrange / Act / Assert with pytest.raises(ValueError, match="Unknown target"): resolve_routing(("generic",), engines={"nope": "duckdb"}) + + +# Per-column-HOMOGENEOUS row strategy: column 0 is always text, columns 1-2 are +# nullable ints. This keeps each column type-homogeneous so the property tests +# exercise normal multiset behaviour rather than the mixed-type sort-key edge. +_interval_rows = st.lists( + st.tuples( + st.text(max_size=5), + st.integers() | st.none(), + st.integers() | st.none(), + ), + max_size=8, +) + + +class TestNormalizeProperties: + """Property-based invariants of `normalize` (Tier-3, Hypothesis).""" + + @given(_interval_rows) + def test_normalize_is_order_invariant(self, rows): + """Test normalize is invariant to input row ordering. + + Given: + An arbitrary list of type-homogeneous rows and a shuffled copy. + When: + normalize() is applied to both orderings. + Then: + The two results should be identical — order-free multiset semantics. + """ + # Arrange + shuffled = list(reversed(rows)) + + # Act / Assert + assert normalize(rows) == normalize(shuffled) + + @given(_interval_rows) + def test_normalize_is_idempotent(self, rows): + """Test normalize applied twice equals normalize applied once. + + Given: + An arbitrary list of type-homogeneous rows. + When: + normalize() is applied, then applied again to its own output. + Then: + The second application should leave the result unchanged. + """ + # Arrange / Act + once = normalize(rows) + twice = normalize(once) + + # Assert + assert once == twice + + +class TestResolveRoutingProperties: + """Property-based totality of `resolve_routing` (Tier-3, Hypothesis).""" + + @given(st.lists(st.text(max_size=10), max_size=5)) + def test_resolve_routing_returns_dict_or_raises_value_error(self, targets): + """Test resolve_routing is total: it returns a dict or raises ValueError. + + Given: + An arbitrary list of target-name strings (known or unknown). + When: + resolve_routing() resolves them. + Then: + It should either return a dict or raise ValueError — never any other + exception type. + """ + # Arrange / Act / Assert + try: + result = resolve_routing(targets) + except ValueError: + return + assert isinstance(result, dict) From caee12a6e0ba1c0ce064a77757f6eb5856f8fab8 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:48:38 -0400 Subject: [PATCH 086/142] test: Trim runner twins and tighten panic/parity assertions Cut the near-duplicate round-trip/quoting/custom-column per-engine twins, keeping the empty-table divergence and cross-runner parity. Tighten the raw empty-partition panic assert with match='index out of bounds', and add empty-table cross-runner parity plus a NULL round-trip parity assertion. --- .../datafusion/test_oracle_runners.py | 206 +++++------------- 1 file changed, 49 insertions(+), 157 deletions(-) diff --git a/tests/integration/datafusion/test_oracle_runners.py b/tests/integration/datafusion/test_oracle_runners.py index 423e6a6..9c8107c 100644 --- a/tests/integration/datafusion/test_oracle_runners.py +++ b/tests/integration/datafusion/test_oracle_runners.py @@ -23,64 +23,7 @@ class TestRunDuckDB: - """`run_duckdb` schema, types, quoting, and empty handling.""" - - def test_run_duckdb_returns_normalized_rows(self): - """Test run_duckdb registers a table and returns normalized rows. - - Given: - A two-row interval table and a plain SELECT. - When: - run_duckdb executes the SQL. - Then: - It should return both rows as sorted plain-tuple values. - """ - # Arrange / Act - rows = run_duckdb( - 'SELECT chrom, start, "end" FROM t', - {"t": [("chr2", 5, 6), ("chr1", 1, 2)]}, - _INTERVAL_COLUMNS, - ) - - # Assert - assert rows == [("chr1", 1, 2), ("chr2", 5, 6)] - - def test_run_duckdb_maps_utf8_and_int64_types(self): - """Test run_duckdb maps utf8 to VARCHAR and int64 to BIGINT. - - Given: - A row whose chrom is a string and coordinates are ints. - When: - run_duckdb executes a SELECT. - Then: - The returned values should preserve str and int Python types. - """ - # Arrange / Act - rows = run_duckdb( - "SELECT chrom, start FROM t", {"t": [("chr1", 1, 2)]}, _INTERVAL_COLUMNS - ) - - # Assert - assert isinstance(rows[0][0], str) - assert isinstance(rows[0][1], int) - - def test_run_duckdb_quotes_reserved_end_column(self): - """Test run_duckdb quotes the reserved ``end`` column. - - Given: - A table with the reserved-word ``end`` column. - When: - run_duckdb selects the quoted column. - Then: - It should return the end value without a parse error. - """ - # Arrange / Act - rows = run_duckdb( - 'SELECT "end" FROM t', {"t": [("chr1", 1, 99)]}, _INTERVAL_COLUMNS - ) - - # Assert - assert rows == [(99,)] + """`run_duckdb` empty-table handling (its ``if rows`` guard).""" def test_run_duckdb_empty_table_returns_zero_rows(self): """Test run_duckdb tolerates an empty table via its ``if rows`` guard. @@ -98,86 +41,9 @@ def test_run_duckdb_empty_table_returns_zero_rows(self): # Assert assert rows == [] - def test_run_duckdb_custom_columns(self): - """Test run_duckdb registers a custom column schema. - - Given: - A table with a custom ``(name, score)`` schema. - When: - run_duckdb selects from it. - Then: - It should map the custom utf8/int64 columns and return the row. - """ - # Arrange / Act - rows = run_duckdb( - "SELECT name, score FROM t", - {"t": [("gene1", 42)]}, - (("name", "utf8"), ("score", "int64")), - ) - - # Assert - assert rows == [("gene1", 42)] - class TestRunDataFusion: - """`run_datafusion` schema, types, quoting, and empty handling.""" - - def test_run_datafusion_returns_normalized_rows(self): - """Test run_datafusion registers a table and returns normalized rows. - - Given: - A two-row interval table and a plain SELECT. - When: - run_datafusion executes the SQL. - Then: - It should return both rows as sorted plain-tuple values. - """ - # Arrange / Act - rows = run_datafusion( - 'SELECT chrom, start, "end" FROM t', - {"t": [("chr2", 5, 6), ("chr1", 1, 2)]}, - _INTERVAL_COLUMNS, - ) - - # Assert - assert rows == [("chr1", 1, 2), ("chr2", 5, 6)] - - def test_run_datafusion_maps_utf8_and_int64_types(self): - """Test run_datafusion maps utf8 to pyarrow utf8 and int64 to int64. - - Given: - A row whose chrom is a string and coordinates are ints. - When: - run_datafusion executes a SELECT. - Then: - The returned values should be plain str and int after coercion. - """ - # Arrange / Act - rows = run_datafusion( - "SELECT chrom, start FROM t", {"t": [("chr1", 1, 2)]}, _INTERVAL_COLUMNS - ) - - # Assert - assert isinstance(rows[0][0], str) - assert isinstance(rows[0][1], int) - - def test_run_datafusion_quotes_reserved_end_column(self): - """Test run_datafusion quotes the reserved ``end`` column. - - Given: - A table with the reserved-word ``end`` column. - When: - run_datafusion selects the quoted column. - Then: - It should return the end value without a parse error. - """ - # Arrange / Act - rows = run_datafusion( - 'SELECT "end" FROM t', {"t": [("chr1", 1, 99)]}, _INTERVAL_COLUMNS - ) - - # Assert - assert rows == [(99,)] + """`run_datafusion` empty-table handling (the synthesized-batch guard).""" def test_run_datafusion_empty_table_returns_zero_rows(self): """Test run_datafusion handles an empty table via a synthesized batch. @@ -197,26 +63,6 @@ def test_run_datafusion_empty_table_returns_zero_rows(self): # Assert assert rows == [] - def test_run_datafusion_custom_columns(self): - """Test run_datafusion registers a custom column schema. - - Given: - A table with a custom ``(name, score)`` schema. - When: - run_datafusion selects from it. - Then: - It should map the custom utf8/int64 columns and return the row. - """ - # Arrange / Act - rows = run_datafusion( - "SELECT name, score FROM t", - {"t": [("gene1", 42)]}, - (("name", "utf8"), ("score", "int64")), - ) - - # Assert - assert rows == [("gene1", 42)] - class TestRawRegisterRecordBatchesEmpty: """Pin the raw DataFusion empty-table panic the runner guards against.""" @@ -245,7 +91,7 @@ def test_raw_register_record_batches_panics_on_empty(self): assert empty_batches == [] # no batches is the panic trigger # Act / Assert - with pytest.raises(BaseException): + with pytest.raises(BaseException, match="index out of bounds"): ctx.register_record_batches("t", [empty_batches]) @@ -273,3 +119,49 @@ def test_runners_agree_on_shape_and_values(self): # Assert assert ddb == df + + def test_runners_agree_on_empty_table(self): + """Test both runners produce identical output for an empty table. + + Given: + An empty table and a trivial SELECT — DuckDB skips the INSERT while + DataFusion synthesizes an empty record batch. + When: + run_duckdb and run_datafusion each execute it. + Then: + Both should return zero rows, proving the two divergent empty-table + paths converge on identical normalized output. + """ + # Arrange + data = {"t": []} + sql = "SELECT chrom FROM t" + + # Act + ddb = run_duckdb(sql, data, _INTERVAL_COLUMNS) + df = run_datafusion(sql, data, _INTERVAL_COLUMNS) + + # Assert + assert ddb == df == [] + + def test_runners_agree_on_null_round_trip(self): + """Test both runners normalize a NULL-bearing column identically. + + Given: + A table with a NULL ``start`` in one row — DataFusion promotes the + int64 column to float64 around the NULL while DuckDB keeps ints. + When: + run_duckdb and run_datafusion each select it. + Then: + Their normalized output should be identical, proving the scalar + coercion collapses the NULL surrogate and the int/float promotion. + """ + # Arrange + data = {"t": [("chr1", None, 2), ("chr2", 3, 4)]} + sql = 'SELECT chrom, start, "end" FROM t' + + # Act + ddb = run_duckdb(sql, data, _INTERVAL_COLUMNS) + df = run_datafusion(sql, data, _INTERVAL_COLUMNS) + + # Assert + assert ddb == df From aa2e08a73d1235aff2e6c82d5c701388ebb74db2 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:48:44 -0400 Subject: [PATCH 087/142] test: Cut dead lane ceremony to the routing-purity guard Remove the capability-truthing tests (they pin supports_lateral/ supports_star_replace flags that production never reads), the version-floor tests (their floors sit below the real pyproject pins so they can never fire), and the import/skip smoke tests (they assert third-party behavior). Keep the genuine resolve_routing-purity guard and mark the module integration so pytest -m integration collects it. --- tests/integration/test_oracle_lane.py | 188 +------------------------- 1 file changed, 7 insertions(+), 181 deletions(-) diff --git a/tests/integration/test_oracle_lane.py b/tests/integration/test_oracle_lane.py index d50c276..2962b0f 100644 --- a/tests/integration/test_oracle_lane.py +++ b/tests/integration/test_oracle_lane.py @@ -1,89 +1,20 @@ -"""Capability-truthing and lane/CI guards for the cross-target oracle (#139, T5). +"""Lane purity guard for the cross-target oracle (#139, T5). -These pin the invariants the oracle's routing and xfails depend on: the -DataFusion target's declared capabilities (no LATERAL, no ``* REPLACE``), that -the engine-free oracle internals are importable and reusable from a non- -DataFusion lane, the skip-path behaviour when an engine is absent, and a -version-floor guard for the optional engines so a too-old DuckDB / DataFusion -fails with a clear message rather than a cryptic runtime error. +This pins the one invariant the oracle's lane layout depends on that nothing +else covers: :func:`resolve_routing` is a *pure* decision that resolves the full +routing map with no engine installed or imported, so non-DataFusion lanes (e.g. +bedtools / coordinate_space) can reuse the engine-free internals. """ -import importlib.util - import pytest -from giql.targets import DataFusionTarget -from giql.targets import DuckDBTarget -from tests.integration._oracle import assert_cross_target -from tests.integration._oracle import normalize from tests.integration._oracle import resolve_routing - -class TestDataFusionCapabilityTruthing: - """The DataFusion capability flags the oracle's xfails rely on.""" - - def test_datafusion_does_not_support_lateral(self): - """Test DataFusionTarget declares no LATERAL support. - - Given: - A DataFusionTarget instance. - When: - Its capabilities are inspected. - Then: - supports_lateral should be False, matching the NEAREST xfail (#142). - """ - # Arrange / Act / Assert - assert DataFusionTarget().capabilities.supports_lateral is False - - def test_datafusion_does_not_support_star_replace(self): - """Test DataFusionTarget declares no ``* REPLACE`` support. - - Given: - A DataFusionTarget instance. - When: - Its capabilities are inspected. - Then: - supports_star_replace should be False (DataFusion has EXCEPT/EXCLUDE - only). - """ - # Arrange / Act / Assert - assert DataFusionTarget().capabilities.supports_star_replace is False - - def test_duckdb_supports_star_replace(self): - """Test DuckDBTarget declares ``* REPLACE`` support as a contrast. - - Given: - A DuckDBTarget instance. - When: - Its capabilities are inspected. - Then: - supports_star_replace should be True, confirming the asymmetry the - DISJOIN note describes is about duplicate output names, not a - ``* REPLACE`` capability gap. - """ - # Arrange / Act / Assert - assert DuckDBTarget().capabilities.supports_star_replace is True +pytestmark = pytest.mark.integration class TestSharedOracleReuse: - """The engine-free internals are importable from any lane.""" - - def test_oracle_internals_reusable_without_engines(self): - """Test the comparison core works imported from a non-DataFusion lane. - - Given: - Two agreeing normalized results assembled with no engine present. - When: - assert_cross_target compares them against a matching expectation. - Then: - It should pass, proving the shared module is reusable outside the - DataFusion lane (e.g. for the bedtools / coordinate_space lanes). - """ - # Arrange - rows = normalize([("chr1", 1, 2)]) - - # Act / Assert - assert_cross_target({"a": rows, "b": rows}, normalize([("chr1", 1, 2)])) + """The engine-free routing decision is reusable from any lane.""" def test_routing_resolves_without_engines(self): """Test resolve_routing is a pure function needing no engine import. @@ -101,108 +32,3 @@ def test_routing_resolves_without_engines(self): # Assert assert set(routing) == {"generic", "datafusion", "duckdb"} - - -class TestEngineSkipPaths: - """Skip / availability behaviour for the optional engines.""" - - def test_duckdb_available_or_skipped(self): - """Test the DuckDB skip-path: present engine imports, absent one skips. - - Given: - The optional DuckDB dependency, which may or may not be installed. - When: - The lane probes for it via importorskip. - Then: - It should import when present (and otherwise skip cleanly), mirroring - the lane's module-level guard. - """ - # Arrange / Act - duckdb = pytest.importorskip("duckdb") - - # Assert - assert hasattr(duckdb, "connect") - - def test_datafusion_available_or_skipped(self): - """Test the DataFusion skip-path: present engine imports, absent skips. - - Given: - The optional DataFusion and pyarrow dependencies. - When: - The lane probes for them via importorskip. - Then: - They should import when present (and otherwise skip cleanly). - """ - # Arrange / Act - datafusion = pytest.importorskip("datafusion") - pytest.importorskip("pyarrow") - - # Assert - assert hasattr(datafusion, "SessionContext") - - -class TestEngineVersionFloor: - """Version-floor guards for the optional engines.""" - - def test_duckdb_meets_version_floor(self): - """Test the installed DuckDB meets the lane's minimum version. - - Given: - The optional DuckDB dependency. - When: - Its version tuple is compared to the floor the IEJoin SQL needs. - Then: - It should be at least the supported floor, failing loudly otherwise. - """ - # Arrange - duckdb = pytest.importorskip("duckdb") - floor = (0, 9) - - # Act - version = tuple(int(p) for p in duckdb.__version__.split(".")[:2]) - - # Assert - assert version >= floor, f"DuckDB {duckdb.__version__} below floor {floor}" - - def test_datafusion_meets_version_floor(self): - """Test the installed DataFusion meets the lane's minimum version. - - Given: - The optional DataFusion dependency. - When: - Its version tuple is compared to the floor the lane was validated on. - Then: - It should be at least the supported floor, failing loudly otherwise. - """ - # Arrange - datafusion = pytest.importorskip("datafusion") - floor = (40, 0) - - # Act - version = tuple(int(p) for p in datafusion.__version__.split(".")[:2]) - - # Assert - assert version >= floor, ( - f"DataFusion {datafusion.__version__} below floor {floor}" - ) - - -class TestLaneCollection: - """A smoke check that the lane and oracle modules collect.""" - - def test_oracle_modules_are_importable(self): - """Test the oracle's importable modules resolve by spec. - - Given: - The non-test ``_oracle`` module and the conftest-backed fixture. - When: - Their import specs are looked up. - Then: - The ``_oracle`` module should be findable, anchoring the lane's - shared-helper layout. - """ - # Arrange / Act - spec = importlib.util.find_spec("tests.integration._oracle") - - # Assert - assert spec is not None From 58b9c1c16c4f37a7886d8b39155948438200fd7f Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 15 Jun 2026 09:56:56 -0400 Subject: [PATCH 088/142] test: Pin nested-replace fix and harden expander coverage Add a discriminating test for the deepest-first nested-replace fix: an outer DISJOIN whose subtree holds an inner DISJOIN, asserting the inner expands into the live tree before the outer replaces it (fails under the old outer-first order). Add a pass-level test that a node missing its resolution now raises ResolutionError, and a register-time test that an object with a non-callable expand attribute is rejected. Route the registry leak guard and clean_registry through the public bool()/len() surface instead of private _expanders, and add a symmetric GIQL_EXPAND opt-in leak guard. Drive the lazy-import subprocess test through the public expand_operators entry point. Assert ctx.node carries the canonical wrapper at dispatch, and add no-residual-operator checks to the walk-location tests. Drop the private-_EXPAND sentinel test (covered by the public opt-out parametrization). Closes #154 --- tests/test_expander.py | 205 ++++++++++++++++++++++++++++++++++------- 1 file changed, 171 insertions(+), 34 deletions(-) diff --git a/tests/test_expander.py b/tests/test_expander.py index afa74af..ad625f1 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -55,12 +55,16 @@ def _expander(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: @pytest.fixture def clean_registry(): - """Isolate the process-wide REGISTRY, restoring its contents afterward.""" - saved = dict(REGISTRY._expanders) + """Isolate the process-wide REGISTRY, leaving it empty afterward. + + The registry is empty at import (the pass ships inert), so a test that opts + in clears on the way out; emptiness is asserted through the public + ``bool()``/``len()`` surface rather than private state. + """ + assert not REGISTRY, "REGISTRY was non-empty entering clean_registry" REGISTRY.clear() yield REGISTRY REGISTRY.clear() - REGISTRY._expanders.update(saved) @pytest.fixture(autouse=True) @@ -71,11 +75,33 @@ def _registry_leak_guard(): every test, so a test that registers without cleaning up (a leak that would silently flip the no-op pass for a later test) fails loudly. Tests that register on the process-wide REGISTRY do so through ``clean_registry``, which - clears on the way out; this guard catches anything that bypasses it. + clears on the way out; this guard catches anything that bypasses it. Both + checks go through the public ``bool()`` surface (A5), not private state. + """ + assert not REGISTRY, "REGISTRY leaked into a test from a prior one" + yield + assert not REGISTRY, "a test leaked a registration into REGISTRY" + + +@pytest.fixture(autouse=True) +def _expand_flag_leak_guard(): + """Assert every operator's GIQL_EXPAND is restored at each test boundary. + + The symmetric partner of the registry leak guard: each operator class ships + opted out (its own GIQL_EXPAND attribute is False), and a test that flips one + via ``_opted_in`` must restore it. A leaked opt-in would silently flip the + no-op pass for a later test, so this catches anything that bypasses the + exception-safe ``_opted_in`` manager. """ - assert not REGISTRY._expanders, "REGISTRY leaked into a test from a prior one" + for op in _OPERATOR_CLASSES: + assert op.__dict__.get("GIQL_EXPAND") is False, ( + f"{op.__name__}.GIQL_EXPAND leaked into a test from a prior one" + ) yield - assert not REGISTRY._expanders, "a test leaked a registration into REGISTRY" + for op in _OPERATOR_CLASSES: + assert op.__dict__.get("GIQL_EXPAND") is False, ( + f"a test leaked a GIQL_EXPAND opt-in on {op.__name__}" + ) class _CountingExpander: @@ -286,6 +312,29 @@ def test_register_rejects_non_callable(self): with pytest.raises(TypeError): registry.register(GenericTarget(), GIQLDisjoin, object()) + def test_register_rejects_object_with_non_callable_expand(self): + """Test that an object whose expand attribute is not callable is rejected. + + Given: + An object exposing an 'expand' attribute that is data, not a method, + so it passes the runtime-checkable protocol's attribute check. + When: + Registering it. + Then: + It should raise TypeError (the registry requires a callable expand, + not merely the attribute's presence). + """ + + # Arrange + class _BadExpander: + expand = "not callable" + + registry = ExpanderRegistry() + + # Act & assert + with pytest.raises(TypeError): + registry.register(GenericTarget(), GIQLDisjoin, _BadExpander()) + class TestExpanderRegistryFallbackGaps: """Edge cases of the registry fallback and op-scoped keying.""" @@ -390,7 +439,7 @@ def test_unregister_absent_key_is_noop(self): registry.unregister(DuckDBTarget(), GIQLDisjoin) assert (DuckDBTarget(), GIQLDisjoin) not in registry - def test_public_teardown_seam_resets_process_registry(self): + def test_public_teardown_seam_resets_process_registry(self, clean_registry): """Test that the public seam tears a custom registration off REGISTRY. Given: @@ -1041,23 +1090,6 @@ def test_operator_class_ships_expand_disabled(self, operator): # Assert assert flag is False - def test_expand_sentinel_is_false(self): - """Test that the shared _EXPAND opt-out sentinel is False. - - Given: - The expressions module's _EXPAND sentinel that every operator's - GIQL_EXPAND defaults to. - When: - Reading it. - Then: - It should be False, the single source the per-class flags inherit. - """ - # Arrange & act - from giql.expressions import _EXPAND - - # Assert - assert _EXPAND is False - class TestOptedInRestoresFlag: """The _opted_in helper restores GIQL_EXPAND even when its body raises.""" @@ -1366,6 +1398,10 @@ def test_expander_sees_canonicalized_operands(self, clean_registry): captured = {} def _capture(node, ctx): + # Snapshot the node's SQL *at dispatch* — canonicalization ran before + # this pass, so a non-canonical target is already wrapped in a + # __giql_canon_ CTE the expander sees in its operand. + captured["node_sql"] = ctx.node.sql(dialect=GIQLDialect) captured["resolution"] = ctx.resolution # Returning the node unchanged leaves the canonical CTE in the tree. return node @@ -1375,11 +1411,11 @@ def _capture(node, ctx): # Act with _opted_in(GIQLDisjoin): - sql = transpile("SELECT * FROM DISJOIN(variants)", tables=[one_based]) + transpile("SELECT * FROM DISJOIN(variants)", tables=[one_based]) # Assert assert captured["resolution"] is not None - assert "__giql_canon_" in sql + assert "__giql_canon_" in captured["node_sql"] def test_replacement_reaches_generator(self, clean_registry): """Test that the expander's replacement node is what the generator renders. @@ -1483,10 +1519,11 @@ def test_walk_reaches_nested_operator(self, clean_registry): # Act with _opted_in(GIQLDisjoin): - pass_.transform(ast) + result = pass_.transform(ast) # Assert assert len(counting.calls) == 1 + assert not list(result.find_all(GIQLDisjoin)) @pytest.mark.parametrize( "query", @@ -1519,10 +1556,11 @@ def test_walk_reaches_operator_in_each_location(self, clean_registry, query): # Act with _opted_in(Intersects): - pass_.transform(ast) + result = pass_.transform(ast) # Assert assert len(counting.calls) == 1 + assert not list(result.find_all(Intersects)) def test_walk_replacement_serializes_through_generator(self, clean_registry): """Test that the pass's replacement serializes cleanly to SQL. @@ -1679,16 +1717,107 @@ def _mint(node, ctx): assert seen[0] != seen[1] assert all(a.startswith(EXPAND_ALIAS_PREFIX) for a in seen) + def test_walk_expands_inner_before_outer_replaces_subtree(self, clean_registry): + """Test that a nested operator expands into the live tree before its ancestor. + + Given: + An outer DISJOIN whose reference subquery contains an inner DISJOIN, + both flagged. The outer's expander returns a fresh node that does NOT + re-attach the inner subtree, and inspects its own subtree so the + recorded order reveals whether the inner had already expanded when the + outer ran. + When: + Running the pass. + Then: + BOTH operators expand and the outer sees the inner already gone — the + deepest-first order expands the inner into the live tree before the + outer replaces it. Under the former outer-first order + the outer replacement detached the still-pending inner, whose later + replace() landed in a discarded subtree, so the inner never reached + the live tree (#154). The recorded order pins the discriminator: the + inner must expand first. + """ + # Arrange + order = [] + tables = _tables(("variants", "genes")) + ast = _prepare( + "SELECT * FROM " + "DISJOIN(variants, reference := (SELECT * FROM DISJOIN(genes)))", + tables, + ) + # Identify the outer node up front: it is the one carrying a descendant + # DISJOIN. Tag by identity so the label survives the inner's replacement. + outer = next( + n for n in ast.find_all(GIQLDisjoin) if list(n.find_all(GIQLDisjoin))[1:] + ) + + def _expander(node, ctx): + if node is outer: + # When the outer runs, the inner must already be gone from the + # outer's subtree (deepest-first expanded and replaced it). Under + # the old outer-first order an un-expanded inner DISJOIN would + # still be present here. + order.append( + "outer_sees_no_inner" + if not list(node.find_all(GIQLDisjoin))[1:] + else "outer_sees_inner" + ) + return exp.column("EXPANDED_outer") + order.append("inner") + return exp.column("EXPANDED_inner") + + clean_registry.register(GenericTarget(), GIQLDisjoin, _expander) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(GIQLDisjoin): + result = pass_.transform(ast) + + # Assert + # The inner expands first, into the live tree, so the outer sees it gone. + assert order == ["inner", "outer_sees_no_inner"] + assert not list(result.find_all(GIQLDisjoin)) + + def test_transform_raises_on_node_missing_resolution(self, clean_registry): + """Test that the pass raises when an operator lacks resolution metadata. + + Given: + A flagged DISJOIN whose pass-1 resolution metadata has been stripped + (an internal invariant violation), with a registered expander. + When: + Running the pass. + Then: + It raises ResolutionError rather than dispatching with a None + resolution (pass 1 must annotate every operator node). + """ + # Arrange + from giql.resolver import META_KEY + from giql.resolver import ResolutionError + + clean_registry.register(GenericTarget(), GIQLDisjoin, _record("x")) + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + for node in ast.find_all(GIQLDisjoin): + node.meta.pop(META_KEY, None) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act & assert + with _opted_in(GIQLDisjoin): + with pytest.raises(ResolutionError): + pass_.transform(ast) + def test_lazy_import_has_no_cycle_in_fresh_process(self): """Test that importing the expander module standalone raises no cycle. Given: A fresh Python process with nothing pre-imported. When: - Importing giql.expander and calling its lazy operator accessor. + Importing giql.expander and running its public expand_operators pass + over a parsed operator AST. Then: - It imports and resolves the nine operator classes without an import - cycle (the lazy _giql_operators import breaks the module cycle). + It imports and runs the inert pass without an import cycle, leaving at + least one operator node intact (the empty registry is a no-op), so the + lazy operator import resolves through a public entry point. """ # Arrange import os @@ -1696,9 +1825,17 @@ def test_lazy_import_has_no_cycle_in_fresh_process(self): import sys code = ( - "import giql.expander as e; " - "ops = e._giql_operators(); " - "assert len(ops) == 9, ops; " + "from giql.expander import expand_operators, ExpanderRegistry; " + "from giql.expressions import GIQLDisjoin; " + "from giql.dialect import GIQLDialect; " + "from giql.table import Table, Tables; " + "from giql.targets import GenericTarget; " + "from sqlglot import parse_one; " + "t = Tables(); t.register('variants', Table('variants')); " + "ast = parse_one('SELECT * FROM DISJOIN(variants)', dialect=GIQLDialect); " + "out = expand_operators(ast, GenericTarget(), t, ExpanderRegistry()); " + "ops = list(out.find_all(GIQLDisjoin)); " + "assert len(ops) >= 1, ops; " "print('ok')" ) # Strip coverage's subprocess auto-start hooks so the child is a genuinely From 0d32f61bbdf5beeb03fff303c2048197d5ccf8ad Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 28 Jun 2026 00:10:59 -0400 Subject: [PATCH 089/142] feat: Add snapshot and restore seam to ExpanderRegistry Provide a public save/restore pair on the expander registry so a caller can capture the current registrations and later re-install exactly that baseline. This lets an isolating test fixture (or a plugin) clear and mutate the process-wide registry around a body without permanently losing the built-in expanders registered at import. snapshot returns a fresh mapping, so mutating it does not affect the registry; restore drops every current entry and re-installs the captured contents. --- src/giql/expander.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/giql/expander.py b/src/giql/expander.py index f93132a..c6558ac 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -253,6 +253,31 @@ def clear(self) -> None: """ self._expanders.clear() + def snapshot(self) -> dict[tuple[Target, type], ExpanderFn]: + """Return a shallow copy of the current registrations. + + The save half of the registry's **public save/restore seam**: a test + fixture (or a plugin) that mutates the process-wide :data:`REGISTRY` + around a body — registering or clearing entries — captures the baseline + with this and hands it back to :meth:`restore` afterward, so the + built-in expanders registered at import survive an isolating fixture + that would otherwise :meth:`clear` them permanently. + + The returned dict is a fresh mapping (mutating it does not affect the + registry), keyed by the same ``(target, operator)`` tuples. + """ + return dict(self._expanders) + + def restore(self, snapshot: dict[tuple[Target, type], ExpanderFn]) -> None: + """Replace all registrations with those captured by :meth:`snapshot`. + + The restore half of the save/restore seam. Drops every current entry and + re-installs exactly the *snapshot* contents, so a fixture can return the + registry to a previously captured baseline regardless of what its body + registered or cleared. + """ + self._expanders = dict(snapshot) + def __contains__(self, key: tuple[Target, type]) -> bool: """Whether an *exact* ``(target, operator)`` entry is registered. From c682c5aa875ccdd2c9f4e11b3f73dffc3b073759 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 28 Jun 2026 00:11:15 -0400 Subject: [PATCH 090/142] refactor: Migrate DISTANCE onto the AST expander pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move DISTANCE generation off the legacy giqldistance_sql string emitter and onto the registry's AST-expansion pass. A new auto-discovering giql.expanders package holds a generic expander that builds the same CASE expression as an AST subtree; transpile imports the package so the process-wide registry is populated before the first transpile. DISTANCE is the proof-of-concept for the expander protocol: it is a single CASE with no joins or per-target divergence, so one generic expander registered for GenericTarget serves every target. The four shapes (unsigned/signed by non-stranded/stranded) and the bedtools closest -d parity offset are preserved verbatim from the deleted emitter. The change is behavior-preserving. Because the CASE is now reserialized by the active target's serializer rather than spliced in as a raw string, the emitted text changes cosmetically only — most visibly the chrom-mismatch guard renders the SQL-standard <> instead of !=. Flip GIQLDistance.GIQL_EXPAND to True and delete the now-dead giqldistance_sql and _distance_operand methods; the shared _generate_distance_case helper stays for NEAREST. --- src/giql/expanders/__init__.py | 19 +++ src/giql/expanders/distance.py | 255 +++++++++++++++++++++++++++++++++ src/giql/expressions.py | 5 +- src/giql/generators/base.py | 80 ----------- src/giql/transpile.py | 1 + 5 files changed, 279 insertions(+), 81 deletions(-) create mode 100644 src/giql/expanders/__init__.py create mode 100644 src/giql/expanders/distance.py diff --git a/src/giql/expanders/__init__.py b/src/giql/expanders/__init__.py new file mode 100644 index 0000000..86bf43e --- /dev/null +++ b/src/giql/expanders/__init__.py @@ -0,0 +1,19 @@ +"""Built-in operator expanders for epic #137. + +Importing this package registers every built-in expander as a side effect: +each submodule decorates its expander(s) with ``@register(...)`` at import +time, and this package imports all of them. The import is wired once (in +:mod:`giql.transpile`) so the process-wide ``REGISTRY`` is populated before the +first transpile. + +New operator modules are picked up automatically: drop a ``.py`` into +this package and it is imported here without editing this file. +""" + +from __future__ import annotations + +import importlib +import pkgutil + +for _module_info in pkgutil.iter_modules(__path__): + importlib.import_module(f"{__name__}.{_module_info.name}") diff --git a/src/giql/expanders/distance.py b/src/giql/expanders/distance.py new file mode 100644 index 0000000..8675121 --- /dev/null +++ b/src/giql/expanders/distance.py @@ -0,0 +1,255 @@ +"""The generic DISTANCE operator expander (epic #137, step / issue #140). + +DISTANCE is the proof-of-concept that validates the expander protocol, the +registry dispatch, and the cross-target result-oracle workflow before the +harder operators migrate. It is the simplest operator — a single ``CASE`` +expression with no joins, CTEs, or per-target divergence — so a single +*generic* expander registered for :class:`~giql.targets.GenericTarget` serves +every target. DISTANCE emits identical SQL on DuckDB, DataFusion, and the +generic baseline, so no per-target override is needed. + +The CASE this expander builds matches +:meth:`giql.generators.base.BaseGIQLGenerator._generate_distance_case` exactly +(bedtools ``closest -d`` semantics): overlapping intervals report ``0``, +book-ended (adjacent) intervals report ``1``, and a raw half-open gap of ``N`` +bases reports ``N + 1``. The ``+ 1`` is applied to the absolute gap magnitude +before any directional sign, so a downstream book-ended pair reports ``+1`` and +an upstream one ``-1`` in signed mode. There are four shapes — the cartesian +product of unsigned/signed and non-stranded/stranded — preserved verbatim from +the legacy emitter. + +Because the returned CASE is reserialized by the active target's serializer +(rather than spliced in as a raw string, as the legacy ``giqldistance_sql`` +emitter did), the emitted text changes cosmetically — most visibly ``!=`` +renders as the SQL-standard ``<>``. The two are semantically identical. +""" + +from __future__ import annotations + +from sqlglot import exp +from sqlglot import parse_one + +from giql.dialect import GIQLDialect +from giql.expander import ExpansionContext +from giql.expander import register +from giql.expressions import GIQLDistance +from giql.resolver import ResolvedColumn +from giql.targets import GenericTarget + +__all__ = ["expand_distance"] + + +def _frag(fragment: str) -> exp.Expression: + """Parse one canonicalized SQL fragment into AST. + + The pass-1 :class:`~giql.resolver.ResolvedColumn` endpoints are SQL string + fragments (e.g. ``a."end"``, or ``'chr1'`` for a literal range) already + canonicalized in place by pass 2, so they are parsed — not rebuilt — back + into AST under the GIQL dialect. + """ + return parse_one(fragment, dialect=GIQLDialect) + + +def _gap(near_start: str, far_end: str) -> exp.Expression: + """Build ``(near_start - far_end + 1)`` — the bedtools-parity gap magnitude. + + Mirrors the legacy ``({start} - {end} + 1)`` fragment: the ``+ 1`` lifts a + book-ended (adjacent) pair from a raw half-open gap of ``0`` to a reported + distance of ``1``. + """ + diff = exp.Sub(this=_frag(near_start), expression=_frag(far_end)) + return exp.paren(exp.Add(this=diff, expression=exp.Literal.number(1))) + + +def _bool_param(param: exp.Expression | None) -> bool: + """Coerce an optional DISTANCE boolean argument to a Python ``bool``. + + Mirrors ``BaseGIQLGenerator._extract_bool_param`` so the expander reads the + ``stranded`` / ``signed`` keyword arguments identically to the legacy + emitter. + """ + if not param: + return False + if isinstance(param, exp.Boolean): + return bool(param.this) + return str(param).upper() in ("TRUE", "1", "YES") + + +def _operand(ctx: ExpansionContext, arg: str, position: str) -> ResolvedColumn: + """Return the resolved column for one DISTANCE interval operand. + + Reads the pass-1 metadata attached to the node. A deferred operand (a + literal range, or an unqualified column the resolver could not resolve) has + no column attached; this raises the historical literal-range diagnostic so + the public error contract is preserved. + """ + resolution = ctx.resolution + if resolution is not None: + resolved = resolution.column(arg) + if resolved is not None: + return resolved + raise ValueError(f"Literal range as {position} argument not yet supported") + + +def _unsigned_distance(col_a: ResolvedColumn, col_b: ResolvedColumn) -> exp.Expression: + """Branch 1: unsigned (absolute) non-stranded distance, returning ``|gap| + 1``.""" + return _wrap_overlap_case( + col_a, + col_b, + downstream=_gap(col_b.start, col_a.end), + upstream=_gap(col_a.start, col_b.end), + ) + + +def _signed_distance(col_a: ResolvedColumn, col_b: ResolvedColumn) -> exp.Expression: + """Branch 2: signed non-stranded distance. + + ``+`` downstream (B after A), ``-`` upstream (B before A). + """ + return _wrap_overlap_case( + col_a, + col_b, + downstream=_gap(col_b.start, col_a.end), + upstream=exp.Neg(this=_gap(col_a.start, col_b.end)), + ) + + +def _stranded_distance( + col_a: ResolvedColumn, col_b: ResolvedColumn, signed: bool +) -> exp.Expression: + """Branches 3 & 4: stranded distance, flipping sign on A's ``-`` strand. + + The downstream and upstream gaps each become a nested ``CASE`` keyed on + ``strand_a``. The ``signed`` flag additionally layers the directional sign + on top of the strand flip, exactly as the legacy emitter's two stranded + branches do. + """ + strand_a = col_a.strand + strand_b = col_b.strand + assert strand_a is not None and strand_b is not None # gated by caller + + down_gap = _gap(col_b.start, col_a.end) + up_gap = _gap(col_a.start, col_b.end) + + if signed: + # Stranded + signed: strand flip AND directional sign. Downstream (B + # after A) is positive by default but flips negative on A's '-' strand; + # upstream (B before A) is negative by default but flips positive on '-'. + downstream = _strand_flip_case( + strand_a, neg=exp.Neg(this=down_gap), pos=down_gap.copy() + ) + upstream = _strand_flip_case( + strand_a, neg=up_gap, pos=exp.Neg(this=up_gap.copy()) + ) + else: + # Stranded but not signed: strand flip only. + downstream = _strand_flip_case( + strand_a, neg=exp.Neg(this=down_gap), pos=down_gap.copy() + ) + upstream = _strand_flip_case( + strand_a, neg=exp.Neg(this=up_gap), pos=up_gap.copy() + ) + + case = _wrap_overlap_case(col_a, col_b, downstream=downstream, upstream=upstream) + # Prepend the strand-validity guards ahead of the overlap guards. The WHEN + # order matters: chrom mismatch, then strand NULL/'.'/'?', then overlap. + return _prepend_strand_guards(case, strand_a, strand_b) + + +def _strand_flip_case( + strand_a: str, neg: exp.Expression, pos: exp.Expression +) -> exp.Expression: + """Build ``CASE WHEN strand_a = '-' THEN neg ELSE pos END``.""" + return ( + exp.Case() + .when(exp.EQ(this=_frag(strand_a), expression=exp.Literal.string("-")), neg) + .else_(pos) + ) + + +def _wrap_overlap_case( + col_a: ResolvedColumn, + col_b: ResolvedColumn, + downstream: exp.Expression, + upstream: exp.Expression, +) -> exp.Expression: + """Build the shared distance CASE skeleton common to all four branches. + + ``CASE WHEN chrom_a != chrom_b THEN NULL WHEN THEN 0 WHEN + end_a <= start_b THEN ELSE END`` — *downstream* is + the B-after-A gap, *upstream* the B-before-A gap (each already carrying any + sign / strand flip). + """ + overlap = exp.and_( + exp.LT(this=_frag(col_a.start), expression=_frag(col_b.end)), + exp.GT(this=_frag(col_a.end), expression=_frag(col_b.start)), + ) + chrom_mismatch = exp.NEQ(this=_frag(col_a.chrom), expression=_frag(col_b.chrom)) + end_a_le_start_b = exp.LTE(this=_frag(col_a.end), expression=_frag(col_b.start)) + return ( + exp.Case() + .when(chrom_mismatch, exp.Null()) + .when(overlap, exp.Literal.number(0)) + .when(end_a_le_start_b, downstream) + .else_(upstream) + ) + + +def _prepend_strand_guards( + case: exp.Case, strand_a: str, strand_b: str +) -> exp.Case: + """Insert the strand-validity WHEN guards after the chrom guard. + + Returns a CASE whose WHEN order is: chrom mismatch -> NULL, either strand + NULL -> NULL, strand_a is '.'/'?' -> NULL, strand_b is '.'/'?' -> NULL, + then the overlap/gap branches from *case*. Distance is undefined for an + unstranded ('.'/'?') or missing strand, matching the legacy emitter. + """ + sa = _frag(strand_a) + sb = _frag(strand_b) + null_guard = exp.condition(exp.Is(this=sa, expression=exp.Null())).or_( + exp.Is(this=sb.copy(), expression=exp.Null()) + ) + a_unstranded = exp.condition( + exp.EQ(this=sa.copy(), expression=exp.Literal.string(".")) + ).or_(exp.EQ(this=sa.copy(), expression=exp.Literal.string("?"))) + b_unstranded = exp.condition( + exp.EQ(this=sb.copy(), expression=exp.Literal.string(".")) + ).or_(exp.EQ(this=sb.copy(), expression=exp.Literal.string("?"))) + + guards = [ + exp.If(this=null_guard, true=exp.Null()), + exp.If(this=a_unstranded, true=exp.Null()), + exp.If(this=b_unstranded, true=exp.Null()), + ] + # The existing WHENs keep their order; the strand guards slot in right after + # the leading chrom-mismatch guard. + existing = case.args["ifs"] + case.set("ifs", existing[:1] + guards + existing[1:]) + return case + + +@register(GenericTarget, GIQLDistance) +def expand_distance(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: + """Expand a ``GIQLDistance`` node into a standard SQL ``CASE`` expression. + + The portable expander registered for every target. Reads ``stranded`` / + ``signed`` from the node and the pass-1 resolved interval operands, then + builds the matching one of the four CASE shapes + (unsigned/signed x non-stranded/stranded). The result replaces the operator + node and is rendered by the active target's serializer. + """ + stranded = _bool_param(node.args.get("stranded")) + signed = _bool_param(node.args.get("signed")) + + col_a = _operand(ctx, "this", "first") + col_b = _operand(ctx, "expression", "second") + + # Strand columns are consumed only in stranded mode, and only when both + # operands actually carry a strand fragment — mirroring the legacy emitter's + # `strand_a is None or strand_b is None` fall-through to the unstranded path. + if stranded and col_a.strand is not None and col_b.strand is not None: + return _stranded_distance(col_a, col_b, signed=signed) + if signed: + return _signed_distance(col_a, col_b) + return _unsigned_distance(col_a, col_b) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index ce939dc..0c6bc80 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -321,7 +321,10 @@ class GIQLDistance(exp.Func): } GIQL_CANONICALIZE = _CANONICALIZE - GIQL_EXPAND = _EXPAND + # Migrated to the registry's AST-expansion path (epic #137, issue #140): the + # generic expander in giql.expanders.distance builds the CASE; the legacy + # giqldistance_sql emitter is gone. + GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 9038369..1a4c949 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -5,7 +5,6 @@ from giql.canonical import decanonical_start from giql.expressions import Contains from giql.expressions import GIQLDisjoin -from giql.expressions import GIQLDistance from giql.expressions import GIQLNearest from giql.expressions import Intersects from giql.expressions import SpatialSetPredicate @@ -493,85 +492,6 @@ def _disjoin_passthrough( f't.* REPLACE ({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' ) - def giqldistance_sql(self, expression: GIQLDistance) -> str: - """Generate SQL CASE expression for DISTANCE function. - - Reads the :class:`~giql.resolver.ResolvedColumn` metadata that - ``ResolveOperatorRefs`` (pass 1) attaches to each interval operand. When - the pass deferred an operand (a literal range, or an unqualified column) - the emitter raises the historical literal-range diagnostic. - - Coordinate canonicalization is owned by ``CanonicalizeCoordinates`` - (pass 2, issue #123): the resolved metadata's endpoints are already - canonicalized in place, so the emitter consumes them verbatim. - - :param expression: - GIQLDistance expression node - :return: - SQL CASE expression string calculating genomic distance - """ - stranded = self._extract_bool_param(expression.args.get("stranded")) - signed = self._extract_bool_param(expression.args.get("signed")) - - col_a = self._distance_operand(expression, "this", "first") - col_b = self._distance_operand(expression, "expression", "second") - - # Strand columns are consumed only in stranded mode (matching the - # historical 3-tuple vs 4-tuple branching in the legacy emitter). - strand_a = col_a.strand if stranded else None - strand_b = col_b.strand if stranded else None - - # Distance math below assumes 0-based half-open. Input canonicalization is - # owned by CanonicalizeCoordinates (pass 2, issue #123): each operand's - # start/end fragments are canonicalized in place by the pass, so the - # emitter consumes them verbatim with no in-emitter canonicalization. The - # returned distance is an encoding-invariant base count, so it needs no - # output de-canonicalization. - - # Generate CASE expression - return self._generate_distance_case( - col_a.chrom, - col_a.start, - col_a.end, - strand_a, - col_b.chrom, - col_b.start, - col_b.end, - strand_b, - stranded=stranded, - signed=signed, - ) - - def _distance_operand( - self, expression: GIQLDistance, arg: str, position: str - ) -> ResolvedColumn: - """Return the :class:`ResolvedColumn` for one DISTANCE interval operand. - - Reads the metadata attached by ``ResolveOperatorRefs`` (pass 1). When the - pass deferred the operand — a literal range or an unqualified column it - could not resolve — no column is attached and this raises the historical - literal-range diagnostic. - - :param expression: - GIQLDistance expression node - :param arg: - The operand arg key (``"this"`` or ``"expression"``) - :param position: - Human-readable operand position for the error message (``"first"`` - or ``"second"``) - :return: - The resolved column operand - :raises ValueError: - If the operand is a literal range rather than a column reference - """ - resolution = expression.meta.get(META_KEY) - if isinstance(resolution, OperatorResolution): - resolved = resolution.column(arg) - if resolved is not None: - return resolved - - raise ValueError(f"Literal range as {position} argument not yet supported") - def _generate_distance_case( self, chrom_a: str, diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 9ef2100..1f806bc 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -11,6 +11,7 @@ from sqlglot import parse_one +import giql.expanders # noqa: F401 from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expander import ExpandOperators From 92b821dbdce84d950765ac2c7852f7a70681224d Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 28 Jun 2026 00:11:26 -0400 Subject: [PATCH 091/142] test: Cover the DISTANCE expander migration and registry seam Run the DISTANCE emitter-level tests through the expansion pass so they exercise the new path, and update their pinned SQL to the reserialized form (<> for the chrom-mismatch guard). The literal-range error tests now assert the diagnostic is raised by the expander rather than the deleted emitter. Rework the registry leak guards in test_expander to treat the import-time built-in registrations as the baseline rather than an empty registry, using the new snapshot and restore seam so isolating fixtures do not wipe the built-ins. Add an _opted_out helper and migrated-vs- unmigrated operator parametrization so a shipped GIQL_EXPAND=True operator can be held as a control, and cover snapshot/restore directly. --- tests/generators/test_base.py | 46 +++--- tests/test_distance_transpilation.py | 22 +-- tests/test_distance_udf.py | 17 ++- tests/test_expander.py | 204 ++++++++++++++++++++++----- 4 files changed, 221 insertions(+), 68 deletions(-) diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index f95e91b..e7f8c69 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -11,31 +11,39 @@ from sqlglot import exp from sqlglot import parse_one +import giql # noqa: F401 (ensures the built-in expanders are registered) from giql import Table from giql import transpile from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect +from giql.expander import ExpandOperators from giql.expressions import GIQLNearest from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables +from giql.targets import GenericTarget def _generate_through_passes(sql: str, tables: Tables) -> str: - """Parse, run normalization passes 1 and 2, then generate SQL. + """Parse, run normalization passes 1-3, then generate SQL. Coordinate canonicalization for operator operands moved out of the emitter and - into the CanonicalizeCoordinates pass (issue #123). Emitter-level tests that - pin canonicalized output must therefore run both passes before generating, - exactly as :func:`giql.transpile.transpile` does, rather than calling - ``generate`` on a bare parsed AST (which would skip canonicalization). This - helper is used where the full ``transpile`` pipeline would otherwise rewrite - the node away (a column-to-column ``INTERSECTS`` is turned into a binned - equi-join before the predicate emitter runs). + into the CanonicalizeCoordinates pass (issue #123), and DISTANCE generation + itself moved onto the registry's AST-expansion pass (epic #137, issue #140). + Emitter-level tests that pin canonicalized / expanded output must therefore + run all three passes before generating, exactly as + :func:`giql.transpile.transpile` does, rather than calling ``generate`` on a + bare parsed AST. The expansion pass only touches operators that opt in + (``GIQL_EXPAND``); operators still on the legacy emitter (NEAREST, the + spatial predicates) pass through untouched. This helper is used where the + full ``transpile`` pipeline would otherwise rewrite the node away (a + column-to-column ``INTERSECTS`` is turned into a binned equi-join before the + predicate emitter runs). """ ast = parse_one(sql, dialect=GIQLDialect) ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) + ast = ExpandOperators(GenericTarget(), tables).transform(ast) return BaseGIQLGenerator(tables=tables).generate(ast) @@ -680,7 +688,7 @@ def test_giqldistance_sql_basic(self, tables_with_two_tables): output = _generate_through_passes(sql, tables_with_two_tables) expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > b."start" ' 'THEN 0 WHEN a."end" <= b."start" ' 'THEN (b."start" - a."end" + 1) ' @@ -703,7 +711,7 @@ def test_giqldistance_sql_stranded(self, tables_with_two_tables): output = _generate_through_passes(sql, tables_with_two_tables) expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' 'WHEN a."strand" IS NULL OR b."strand" IS NULL THEN NULL ' "WHEN a.\"strand\" = '.' OR a.\"strand\" = '?' THEN NULL " "WHEN b.\"strand\" = '.' OR b.\"strand\" = '?' THEN NULL " @@ -734,7 +742,7 @@ def test_giqldistance_sql_signed(self, tables_with_two_tables): output = _generate_through_passes(sql, tables_with_two_tables) expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > b."start" ' 'THEN 0 WHEN a."end" <= b."start" ' 'THEN (b."start" - a."end" + 1) ' @@ -758,7 +766,7 @@ def test_giqldistance_sql_stranded_and_signed(self, tables_with_two_tables): output = _generate_through_passes(sql, tables_with_two_tables) expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' 'WHEN a."strand" IS NULL OR b."strand" IS NULL THEN NULL ' "WHEN a.\"strand\" = '.' OR a.\"strand\" = '?' THEN NULL " "WHEN b.\"strand\" = '.' OR b.\"strand\" = '?' THEN NULL " @@ -804,7 +812,7 @@ def test_giqldistance_canonicalizes_closed_ends_apart_from_gap_parity( # Assert expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' 'WHEN a."start" < (b."end" + 1) ' 'AND (a."end" + 1) > b."start" THEN 0 ' 'WHEN (a."end" + 1) <= b."start" ' @@ -966,10 +974,10 @@ def test_giqldistance_sql_literal_first_arg_error(self, tables_with_two_tables): ast = resolve_operator_refs(ast, tables_with_two_tables) ast = canonicalize_coordinates(ast) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) + expander = ExpandOperators(GenericTarget(), tables_with_two_tables) with pytest.raises(ValueError, match="Literal range as first argument"): - generator.generate(ast) + expander.transform(ast) def test_giqldistance_sql_literal_second_arg_error(self, tables_with_two_tables): """ @@ -982,10 +990,10 @@ def test_giqldistance_sql_literal_second_arg_error(self, tables_with_two_tables) ast = resolve_operator_refs(ast, tables_with_two_tables) ast = canonicalize_coordinates(ast) - generator = BaseGIQLGenerator(tables=tables_with_two_tables) + expander = ExpandOperators(GenericTarget(), tables_with_two_tables) with pytest.raises(ValueError, match="Literal range as second argument"): - generator.generate(ast) + expander.transform(ast) def test_giqlnearest_sql_missing_outer_table_error( self, tables_with_peaks_and_genes @@ -1630,7 +1638,7 @@ def test_giqldistance_should_canonicalize_table_columns_for_each_convention( # Assert expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " f"ELSE ({start_a} - {end_b} + 1) END AS dist " @@ -1664,7 +1672,7 @@ def test_giqldistance_should_canonicalize_each_side_when_conventions_differ( # Assert expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > (b."start" - 1) THEN 0 ' 'WHEN a."end" <= (b."start" - 1) ' 'THEN ((b."start" - 1) - a."end" + 1) ' diff --git a/tests/test_distance_transpilation.py b/tests/test_distance_transpilation.py index 944a3db..7169d87 100644 --- a/tests/test_distance_transpilation.py +++ b/tests/test_distance_transpilation.py @@ -5,27 +5,33 @@ from sqlglot import parse_one +import giql # noqa: F401 (ensures the built-in expanders are registered) from giql import transpile from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect +from giql.expander import ExpandOperators from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables +from giql.targets import GenericTarget def _generate(sql: str, tables: Tables | None = None) -> str: - """Parse, run normalization passes 1 and 2, then generate SQL. + """Parse, run normalization passes 1-3, then generate SQL. DISTANCE operand resolution and coordinate canonicalization moved out of the emitter and into the ResolveOperatorRefs / CanonicalizeCoordinates passes - (epic #114, issues #119 / #123). Emitter-level tests must run both passes - before generating, exactly as :func:`giql.transpile.transpile` does, rather - than calling ``generate`` on a bare parsed AST. + (epic #114, issues #119 / #123). DISTANCE generation itself then moved onto + the registry's AST-expansion pass (epic #137, issue #140), so the operator + node must be expanded before generation too. Emitter-level tests run all + three passes, exactly as :func:`giql.transpile.transpile` does, rather than + calling ``generate`` on a bare parsed AST. """ tables = tables or Tables() ast = parse_one(sql, dialect=GIQLDialect) ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) + ast = ExpandOperators(GenericTarget(), tables).transform(ast) return BaseGIQLGenerator(tables=tables).generate(ast) @@ -45,7 +51,7 @@ def test_distance_transpilation_duckdb(self): output = _generate(sql) - expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" + expected = """SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" assert output == expected, f"Expected:\n{expected}\n\nGot:\n{output}" @@ -62,7 +68,7 @@ def test_distance_transpilation_sqlite(self): output = _generate(sql) - expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a, features_b AS b""" + expected = """SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a, features_b AS b""" assert output == expected, f"Expected:\n{expected}\n\nGot:\n{output}" @@ -79,7 +85,7 @@ def test_distance_transpilation_postgres(self): output = _generate(sql) - expected = """SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" + expected = """SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ELSE (a."start" - b."end" + 1) END AS dist FROM features_a AS a CROSS JOIN features_b AS b""" assert output == expected, f"Expected:\n{expected}\n\nGot:\n{output}" @@ -119,7 +125,7 @@ def test_distance_transpilation_signed_duckdb(self): # Signed distance: upstream (B before A) returns negative, # downstream (B after A) returns positive expected = ( - 'SELECT CASE WHEN a."chrom" != b."chrom" THEN NULL ' + 'SELECT CASE WHEN a."chrom" <> b."chrom" THEN NULL ' 'WHEN a."start" < b."end" AND a."end" > b."start" THEN 0 ' 'WHEN a."end" <= b."start" THEN (b."start" - a."end" + 1) ' 'ELSE -(a."start" - b."end" + 1) END AS dist ' diff --git a/tests/test_distance_udf.py b/tests/test_distance_udf.py index 65f9c19..bf12ef1 100644 --- a/tests/test_distance_udf.py +++ b/tests/test_distance_udf.py @@ -8,25 +8,32 @@ import pytest from sqlglot import parse_one +import giql # noqa: F401 (ensures the built-in expanders are registered) from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect +from giql.expander import ExpandOperators from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables +from giql.targets import GenericTarget def _generate(sql: str) -> str: - """Parse, run normalization passes 1 and 2, then generate SQL. + """Parse, run normalization passes 1-3, then generate SQL. DISTANCE operand resolution and coordinate canonicalization moved out of the emitter and into the ResolveOperatorRefs / CanonicalizeCoordinates passes - (epic #114, issues #119 / #123). These behavioral tests must run both passes - before generating, exactly as :func:`giql.transpile.transpile` does, rather - than calling ``generate`` on a bare parsed AST. + (epic #114, issues #119 / #123), and DISTANCE generation itself moved onto + the registry's AST-expansion pass (epic #137, issue #140). These behavioral + tests must run all three passes before generating, exactly as + :func:`giql.transpile.transpile` does, rather than calling ``generate`` on a + bare parsed AST. """ + tables = Tables() ast = parse_one(sql, dialect=GIQLDialect) - ast = resolve_operator_refs(ast, Tables()) + ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) + ast = ExpandOperators(GenericTarget(), tables).transform(ast) return BaseGIQLGenerator().generate(ast) diff --git a/tests/test_expander.py b/tests/test_expander.py index ad625f1..a84077b 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -53,34 +53,48 @@ def _expander(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: return _expander +#: The registry contents at import — the built-in expanders registered by +#: ``giql.expanders`` for already-migrated operators (DISJOIN as of #143). The +#: leak guards and ``clean_registry`` treat this as the baseline rather than an +#: empty registry, so the real built-in registrations survive isolating fixtures +#: and a leaking test is still caught against the true baseline. +_REGISTRY_BASELINE = REGISTRY.snapshot() + + @pytest.fixture def clean_registry(): - """Isolate the process-wide REGISTRY, leaving it empty afterward. + """Isolate the process-wide REGISTRY, restoring its baseline afterward. - The registry is empty at import (the pass ships inert), so a test that opts - in clears on the way out; emptiness is asserted through the public - ``bool()``/``len()`` surface rather than private state. + Saves the import-time baseline (the built-in expanders), empties the registry + so a test sees only what it registers, and restores the baseline on the way + out through the public ``snapshot()``/``restore()`` seam — so a test that + registers a stand-in expander cannot leak it, and the built-in registrations + survive this fixture's isolation. """ - assert not REGISTRY, "REGISTRY was non-empty entering clean_registry" + saved = REGISTRY.snapshot() REGISTRY.clear() yield REGISTRY - REGISTRY.clear() + REGISTRY.restore(saved) @pytest.fixture(autouse=True) def _registry_leak_guard(): - """Assert the process-wide REGISTRY is empty at each test boundary. - - A leak guard: the registry is empty at import and must return to empty after - every test, so a test that registers without cleaning up (a leak that would - silently flip the no-op pass for a later test) fails loudly. Tests that - register on the process-wide REGISTRY do so through ``clean_registry``, which - clears on the way out; this guard catches anything that bypasses it. Both - checks go through the public ``bool()`` surface (A5), not private state. + """Assert the process-wide REGISTRY matches its baseline at each boundary. + + A leak guard: the registry holds the built-in expanders at import and must + return to exactly that baseline after every test, so a test that registers + without cleaning up (a leak that would silently change dispatch for a later + test) fails loudly. Tests that mutate the process-wide REGISTRY do so through + ``clean_registry``, which restores the baseline on the way out; this guard + catches anything that bypasses it. """ - assert not REGISTRY, "REGISTRY leaked into a test from a prior one" + assert REGISTRY.snapshot() == _REGISTRY_BASELINE, ( + "REGISTRY differed from its baseline entering a test" + ) yield - assert not REGISTRY, "a test leaked a registration into REGISTRY" + assert REGISTRY.snapshot() == _REGISTRY_BASELINE, ( + "a test leaked a registration into REGISTRY" + ) @pytest.fixture(autouse=True) @@ -88,18 +102,19 @@ def _expand_flag_leak_guard(): """Assert every operator's GIQL_EXPAND is restored at each test boundary. The symmetric partner of the registry leak guard: each operator class ships - opted out (its own GIQL_EXPAND attribute is False), and a test that flips one - via ``_opted_in`` must restore it. A leaked opt-in would silently flip the - no-op pass for a later test, so this catches anything that bypasses the - exception-safe ``_opted_in`` manager. + a shipped GIQL_EXPAND default (``True`` for a migrated operator like DISJOIN, + ``False`` otherwise), and a test that flips one via ``_opted_in`` must restore + it. A leaked flip would silently change the pass for a later test, so this + catches anything that bypasses the exception-safe ``_opted_in`` manager by + comparing against each operator's shipped default rather than a blanket False. """ for op in _OPERATOR_CLASSES: - assert op.__dict__.get("GIQL_EXPAND") is False, ( + assert op.__dict__.get("GIQL_EXPAND") is _SHIPPED_EXPAND_FLAGS[op], ( f"{op.__name__}.GIQL_EXPAND leaked into a test from a prior one" ) yield for op in _OPERATOR_CLASSES: - assert op.__dict__.get("GIQL_EXPAND") is False, ( + assert op.__dict__.get("GIQL_EXPAND") is _SHIPPED_EXPAND_FLAGS[op], ( f"a test leaked a GIQL_EXPAND opt-in on {op.__name__}" ) @@ -468,6 +483,55 @@ def _expander(node, ctx): assert REGISTRY.resolve(DuckDBTarget(), GIQLDisjoin) is None assert (DuckDBTarget(), GIQLDisjoin) not in REGISTRY + def test_snapshot_is_independent_of_later_registrations(self): + """Test that a snapshot does not observe registrations made after it. + + Given: + A registry with one entry, captured by snapshot. + When: + A second entry is registered after the snapshot is taken. + Then: + The snapshot should still hold only the first entry (it is a copy, + not a live view). + """ + # Arrange + registry = ExpanderRegistry() + registry.register(DuckDBTarget(), GIQLDisjoin, _record("first")) + + # Act + saved = registry.snapshot() + registry.register(GenericTarget(), Intersects, _record("second")) + + # Assert + assert (DuckDBTarget(), GIQLDisjoin) in saved + assert (GenericTarget(), Intersects) not in saved + + def test_restore_replaces_entries_with_snapshot_contents(self): + """Test that restore returns the registry to a captured snapshot. + + Given: + A snapshot of a registry with one entry, after which the registry is + cleared and a different entry registered. + When: + Restoring the snapshot. + Then: + The original entry should resolve again and the post-snapshot entry + should be gone. + """ + # Arrange + registry = ExpanderRegistry() + registry.register(DuckDBTarget(), GIQLDisjoin, _record("original")) + saved = registry.snapshot() + registry.clear() + registry.register(GenericTarget(), Intersects, _record("transient")) + + # Act + registry.restore(saved) + + # Assert + assert (DuckDBTarget(), GIQLDisjoin) in registry + assert (GenericTarget(), Intersects) not in registry + class TestRegisterDecorator: """Tests for the @register extension-hook decorator.""" @@ -956,7 +1020,8 @@ def test_transform_skips_unflagged_operator(self, clean_registry): Given: An expander registered for (GenericTarget, GIQLDisjoin) but the - operator's GIQL_EXPAND flag left at its default False. + operator's GIQL_EXPAND flag held off (DISJOIN ships it on, so the + control opts it out to isolate the per-type gate). When: Running the pass. Then: @@ -968,8 +1033,9 @@ def test_transform_skips_unflagged_operator(self, clean_registry): ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - # Act (GIQL_EXPAND is False by default — no opt-in context) - result = pass_.transform(ast) + # Act + with _opted_out(GIQLDisjoin): + result = pass_.transform(ast) # Assert assert list(result.find_all(GIQLDisjoin)) @@ -1022,14 +1088,18 @@ def test_expand_operators_is_identity_when_registry_empty(self): assert list(result.find_all(GIQLDisjoin)) def test_transpile_sql_unchanged_with_pass_inert(self): - """Test that transpile output is byte-identical with the pass inert. + """Test that transpile output is byte-identical for an unmigrated operator. Given: - A DISJOIN query, the default (empty) registry, and no operator flagged. + A DISJOIN query (an operator not migrated onto the pass, so its + GIQL_EXPAND is False and no expander resolves), with the default + registry. When: - Transpiling with the wired-in pass versus a pass-bypassed reference. + Transpiling with the wired-in pass versus a pass-bypassed reference + (its legacy emitter run directly). Then: - The SQL should match exactly and carry no expander alias prefix. + The SQL should match exactly and carry no expander alias prefix — the + pass is inert for any operator that has not been migrated. """ # Arrange query = "SELECT * FROM DISJOIN(variants)" @@ -1069,20 +1139,39 @@ def test_transpile_sql_unchanged_with_pass_inert(self): GIQLMerge, ) +#: Each operator's shipped GIQL_EXPAND default, captured from its own class dict +#: at import. A migrated operator (DISJOIN, #143) ships ``True``; the rest ship +#: ``False`` until their migrations land. The flag leak guard restores to these +#: shipped values rather than a blanket ``False``. +_SHIPPED_EXPAND_FLAGS = {op: op.__dict__.get("GIQL_EXPAND") for op in _OPERATOR_CLASSES} + + +#: Operators migrated onto the ExpandOperators pass — they ship GIQL_EXPAND=True. +_MIGRATED_OPERATORS = tuple( + op for op in _OPERATOR_CLASSES if op.__dict__.get("GIQL_EXPAND") is True +) +#: Operators not yet migrated — they ship GIQL_EXPAND=False. +_UNMIGRATED_OPERATORS = tuple( + op for op in _OPERATOR_CLASSES if op not in _MIGRATED_OPERATORS +) + class TestOperatorOptOut: - """Every operator ships opted out of the ExpandOperators pass at this step.""" + """Migrated operators opt into the pass; the rest still ship opted out.""" - @pytest.mark.parametrize("operator", _OPERATOR_CLASSES, ids=lambda c: c.__name__) + @pytest.mark.parametrize( + "operator", _UNMIGRATED_OPERATORS, ids=lambda c: c.__name__ + ) def test_operator_class_ships_expand_disabled(self, operator): - """Test that each operator class ships GIQL_EXPAND=False. + """Test that each unmigrated operator class ships GIQL_EXPAND=False. Given: - One of the nine GIQL operator expression classes. + A GIQL operator expression class that has not been migrated onto the + ExpandOperators pass. When: Reading its GIQL_EXPAND class attribute. Then: - It should be False (no operator opts into expansion yet). + It should be False (the operator still uses the legacy emitter). """ # Arrange & act flag = operator.GIQL_EXPAND @@ -1090,6 +1179,27 @@ def test_operator_class_ships_expand_disabled(self, operator): # Assert assert flag is False + @pytest.mark.parametrize( + "operator", _MIGRATED_OPERATORS, ids=lambda c: c.__name__ + ) + def test_operator_class_ships_expand_enabled(self, operator): + """Test that each migrated operator class ships GIQL_EXPAND=True. + + Given: + A GIQL operator expression class migrated onto the ExpandOperators + pass (DISJOIN, #143). + When: + Reading its GIQL_EXPAND class attribute. + Then: + It should be True (the operator expands through its registered + expander instead of the deleted legacy emitter). + """ + # Arrange & act + flag = operator.GIQL_EXPAND + + # Assert + assert flag is True + class TestOptedInRestoresFlag: """The _opted_in helper restores GIQL_EXPAND even when its body raises.""" @@ -1672,8 +1782,8 @@ def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): ) pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - # Act - with _opted_in(Intersects): + # Act (DISJOIN ships flagged, so opt it out to hold it as the control) + with _opted_in(Intersects), _opted_out(GIQLDisjoin): result = pass_.transform(ast) # Assert @@ -1888,3 +1998,25 @@ def __enter__(self): def __exit__(self, *exc): self._operator.GIQL_EXPAND = self._prior return False + + +class _opted_out: + """Context manager opting an operator class out of GIQL_EXPAND for a test. + + The complement of :class:`_opted_in`: used by a control test that needs a + *migrated* operator (DISJOIN ships GIQL_EXPAND=True) to behave as if + unflagged, so the test can prove the pass gates per-type without the + operator's shipped opt-in interfering. Restores the prior flag on exit. + """ + + def __init__(self, operator: type) -> None: + self._operator = operator + self._prior = operator.__dict__.get("GIQL_EXPAND", False) + + def __enter__(self): + self._operator.GIQL_EXPAND = False + return self._operator + + def __exit__(self, *exc): + self._operator.GIQL_EXPAND = self._prior + return False From 851f3bb493add2fdd852e167b21b68aea561b215 Mon Sep 17 00:00:00 2001 From: Conrad Date: Sun, 28 Jun 2026 19:25:37 -0400 Subject: [PATCH 092/142] test: Address PR #156 review findings for DISTANCE Mark the DuckDB-executing distance UDF tests as integration, add a drift-guard parity test between the AST expander and the retained _generate_distance_case, a cross-target byte-identity test, and a Hypothesis property test for the distance invariants. Hoist the shared downstream branch in the stranded CASE, align helper docstrings, and refresh stale giqldistance_sql references. Make the registry docstrings mechanistic, restore the registry in place, harden expander auto-discovery, and key the opt-out control on a dynamically derived migrated operator. --- src/giql/expander.py | 88 +++++++++++---- src/giql/expanders/__init__.py | 8 ++ src/giql/expanders/distance.py | 144 +++++++++++++++++++++---- src/giql/generators/base.py | 11 ++ src/giql/transpile.py | 15 +-- tests/generators/test_base.py | 8 +- tests/test_distance_udf.py | 189 +++++++++++++++++++++++++++++++++ tests/test_expander.py | 50 +++++---- tests/test_transpile.py | 61 ++++++++++- 9 files changed, 495 insertions(+), 79 deletions(-) diff --git a/src/giql/expander.py b/src/giql/expander.py index c6558ac..3b5a900 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -39,10 +39,11 @@ i.e. a ``(target, op)`` or ``(generic, op)`` expander is registered. Otherwise it falls through to the legacy ``*_sql`` emitter on -:class:`giql.generators.base.BaseGIQLGenerator`. As of this issue **no operator -sets ``GIQL_EXPAND`` and the registry is empty, so the pass is a strict no-op**: -no node is touched and the emitted SQL is byte-identical. Each later migration PR -(epic #137 steps 4-9) registers a generic expander, flips one operator's +:class:`giql.generators.base.BaseGIQLGenerator`. The built-in expanders register +at import time via :mod:`giql.expanders`; the pass rewrites a node only when +``GIQL_EXPAND=True`` **and** an expander resolves for ``(active target, operator +type)``, and is a no-op for any operator that is unflagged or has no registered +expander. A migration PR registers an expander, flips one operator's ``GIQL_EXPAND`` flag, and deletes that operator's ``*_sql`` method. """ @@ -149,6 +150,14 @@ class OperatorExpander(Protocol): a registered object satisfies it. A plain function is *not* an ``OperatorExpander`` (it has no ``expand`` method); register one by wrapping it (see :func:`register`, which accepts either form). + + An expander is **node-local**: ``expand(node, ctx) -> exp.Expression`` sees + one operator node and returns the expression that replaces it in place. It + cannot express a whole-query rewrite such as the INTERSECTS IEJoin fold, + which restructures the surrounding query (joins, CTEs) rather than a single + node. That fold is therefore deferred — it would need a separate + query-level mechanism — and is handled by the pre-pass join transformers, not + by an expander. """ def expand(self, node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: ... @@ -215,6 +224,16 @@ def register( expander : OperatorExpander | ExpanderFn The expander object or function. A later registration for the same key replaces an earlier one (last-write-wins override). + + Notes + ----- + Registering a *non-generic* ``(target, operator)`` expander where the + operator has a built-in whole-query join rewrite (notably + :class:`~giql.expressions.Intersects`, whose binned equi-join / DuckDB + IEJoin transformers run before expansion) signals that this expander + assumes responsibility for that rewrite: the built-in join transformers + are bypassed for that target so the operator node flows untouched into + :class:`ExpandOperators`. See :meth:`has_override`. """ self._expanders[(target, operator)] = _as_callable(expander) @@ -223,6 +242,12 @@ def resolve(self, target: Target, operator: type) -> ExpanderFn | None: Tries the exact ``(target, op)`` entry, then the ``(GenericTarget(), op)`` fallback, then ``None`` (legacy emitter). + + A non-generic exact ``(target, op)`` entry is also a *join-rewrite + override* for operators with a built-in whole-query join rewrite (notably + :class:`~giql.expressions.Intersects`): registering one bypasses the + built-in binned / IEJoin transformers for that target (see + :meth:`register` and :meth:`has_override`). """ fn = self._expanders.get((target, operator)) if fn is not None: @@ -233,6 +258,19 @@ def resolve(self, target: Target, operator: type) -> ExpanderFn | None: return fn return None + def has_override(self, target: Target, operator: type) -> bool: + """Whether an exact non-generic override supersedes built-in handling. + + Returns ``True`` only when *target* is not :class:`~giql.targets.GenericTarget` + and an exact ``(target, operator)`` entry is registered. Such an entry is a + target-specific override that supersedes built-in handling for that target + (e.g. it takes responsibility for the whole-query join rewrite that the + built-in transformers would otherwise perform); the portable + ``(GenericTarget(), operator)`` fallback is *not* an override and does not + count here. + """ + return target != GenericTarget() and (target, operator) in self._expanders + def unregister(self, target: Target, operator: type) -> None: """Drop the ``(target, operator)`` entry if present. @@ -256,12 +294,12 @@ def clear(self) -> None: def snapshot(self) -> dict[tuple[Target, type], ExpanderFn]: """Return a shallow copy of the current registrations. - The save half of the registry's **public save/restore seam**: a test - fixture (or a plugin) that mutates the process-wide :data:`REGISTRY` - around a body — registering or clearing entries — captures the baseline - with this and hands it back to :meth:`restore` afterward, so the - built-in expanders registered at import survive an isolating fixture - that would otherwise :meth:`clear` them permanently. + The save half of a save/restore seam used primarily for test baseline + isolation (and which may serve a plugin that mutates the process-wide + :data:`REGISTRY` around a body): capture the baseline with this and hand + it back to :meth:`restore` afterward, so the built-in expanders + registered at import survive an isolating fixture that would otherwise + :meth:`clear` them permanently. It is not a committed plugin API. The returned dict is a fresh mapping (mutating it does not affect the registry), keyed by the same ``(target, operator)`` tuples. @@ -271,12 +309,14 @@ def snapshot(self) -> dict[tuple[Target, type], ExpanderFn]: def restore(self, snapshot: dict[tuple[Target, type], ExpanderFn]) -> None: """Replace all registrations with those captured by :meth:`snapshot`. - The restore half of the save/restore seam. Drops every current entry and - re-installs exactly the *snapshot* contents, so a fixture can return the - registry to a previously captured baseline regardless of what its body - registered or cleared. + The restore half of the save/restore seam (test baseline isolation; may + also serve a plugin, but is not a committed plugin API). Drops every + current entry and re-installs exactly the *snapshot* contents, so a + fixture can return the registry to a previously captured baseline + regardless of what its body registered or cleared. """ - self._expanders = dict(snapshot) + self._expanders.clear() + self._expanders.update(snapshot) def __contains__(self, key: tuple[Target, type]) -> bool: """Whether an *exact* ``(target, operator)`` entry is registered. @@ -305,8 +345,9 @@ def __bool__(self) -> bool: #: The process-wide registry the :func:`register` decorator writes to and the -#: :class:`ExpandOperators` pass reads from. Empty as of this issue, so the pass -#: is a strict no-op. +#: :class:`ExpandOperators` pass reads from. The built-in expanders register into +#: it at import time via :mod:`giql.expanders`; the pass rewrites a node only when +#: an expander resolves here (and the operator is flagged ``GIQL_EXPAND``). REGISTRY = ExpanderRegistry() @@ -405,9 +446,10 @@ class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for ``(target, operator type)`` through its fallback chain; otherwise the node is left untouched and the legacy ``*_sql`` emitter handles it. - The pass mutates and returns *expression* in place. **With no operator - flagged and an empty registry it is a strict no-op** and the emitted SQL is - byte-identical, so the existing suite is the migration oracle. + The pass mutates and returns *expression* in place. It touches only nodes + whose operator is flagged ``GIQL_EXPAND`` and resolves an expander; for every + other operator it is a no-op, leaving the emitted SQL byte-identical, so the + existing suite is the migration oracle. Parameters ---------- @@ -424,9 +466,9 @@ class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for Returns ------- exp.Expression - The same *expression*, with opted-in operator nodes replaced by their - target-specific expansions (none, while every flag is off / the registry - is empty). + The same *expression*, with each opted-in operator node that resolves an + expander replaced by its target-specific expansion; nodes that are + unflagged or resolve no expander are left untouched. """ reg = registry if registry is not None else REGISTRY operators = _giql_operators() diff --git a/src/giql/expanders/__init__.py b/src/giql/expanders/__init__.py index 86bf43e..042a07d 100644 --- a/src/giql/expanders/__init__.py +++ b/src/giql/expanders/__init__.py @@ -8,6 +8,12 @@ New operator modules are picked up automatically: drop a ``.py`` into this package and it is imported here without editing this file. + +Modules whose name starts with ``_`` are skipped (private helpers, not +expanders). Submodules import in :func:`pkgutil.iter_modules` order, which sets +last-write-wins resolution-order precedence for overlapping registrations; an +import error here aborts the whole package import by design (a broken built-in +expander must not be silently skipped). """ from __future__ import annotations @@ -16,4 +22,6 @@ import pkgutil for _module_info in pkgutil.iter_modules(__path__): + if _module_info.name.startswith("_"): + continue importlib.import_module(f"{__name__}.{_module_info.name}") diff --git a/src/giql/expanders/distance.py b/src/giql/expanders/distance.py index 8675121..530b56e 100644 --- a/src/giql/expanders/distance.py +++ b/src/giql/expanders/distance.py @@ -46,18 +46,40 @@ def _frag(fragment: str) -> exp.Expression: fragments (e.g. ``a."end"``, or ``'chr1'`` for a literal range) already canonicalized in place by pass 2, so they are parsed — not rebuilt — back into AST under the GIQL dialect. + + Parameters + ---------- + fragment : str + A canonicalized SQL fragment (a column reference or literal). + + Returns + ------- + exp.Expression + The parsed fragment as a sqlglot AST node. """ return parse_one(fragment, dialect=GIQLDialect) -def _gap(near_start: str, far_end: str) -> exp.Expression: - """Build ``(near_start - far_end + 1)`` — the bedtools-parity gap magnitude. +def _gap(minuend: str, subtrahend: str) -> exp.Expression: + """Build ``(minuend - subtrahend + 1)`` — the bedtools-parity gap magnitude. Mirrors the legacy ``({start} - {end} + 1)`` fragment: the ``+ 1`` lifts a book-ended (adjacent) pair from a raw half-open gap of ``0`` to a reported distance of ``1``. + + Parameters + ---------- + minuend : str + The SQL fragment subtracted *from* (the left operand of the ``-``). + subtrahend : str + The SQL fragment subtracted (the right operand of the ``-``). + + Returns + ------- + exp.Expression + The parenthesized ``(minuend - subtrahend + 1)`` AST. """ - diff = exp.Sub(this=_frag(near_start), expression=_frag(far_end)) + diff = exp.Sub(this=_frag(minuend), expression=_frag(subtrahend)) return exp.paren(exp.Add(this=diff, expression=exp.Literal.number(1))) @@ -67,6 +89,16 @@ def _bool_param(param: exp.Expression | None) -> bool: Mirrors ``BaseGIQLGenerator._extract_bool_param`` so the expander reads the ``stranded`` / ``signed`` keyword arguments identically to the legacy emitter. + + Parameters + ---------- + param : exp.Expression | None + The ``stranded`` or ``signed`` argument node, or ``None`` if absent. + + Returns + ------- + bool + The coerced Python boolean (``False`` when the argument is absent). """ if not param: return False @@ -82,7 +114,30 @@ def _operand(ctx: ExpansionContext, arg: str, position: str) -> ResolvedColumn: literal range, or an unqualified column the resolver could not resolve) has no column attached; this raises the historical literal-range diagnostic so the public error contract is preserved. + + Parameters + ---------- + ctx : ExpansionContext + The expansion context carrying the node's pass-1 resolution. + arg : str + The operand slot key (``"this"`` or ``"expression"``). + position : str + Human-readable operand position (``"first"`` / ``"second"``) for the + diagnostic message. + + Returns + ------- + ResolvedColumn + The resolved column metadata for the operand. + + Raises + ------ + ValueError + If the operand was deferred (a literal range or unresolved column). """ + # TODO(#146): this read-required-column-or-raise pattern is duplicated across + # expanders; hoist it to a shared ``ExpansionContext.require_column`` helper + # once a second expander needs it. resolution = ctx.resolution if resolution is not None: resolved = resolution.column(arg) @@ -131,21 +186,21 @@ def _stranded_distance( down_gap = _gap(col_b.start, col_a.end) up_gap = _gap(col_a.start, col_b.end) + # Downstream (B after A) is identical across the signed and unsigned arms: + # positive by default, flipped negative on A's '-' strand. Only *upstream* + # differs between the arms, so hoist downstream and compute upstream per arm. + downstream = _strand_flip_case( + strand_a, neg=exp.Neg(this=down_gap), pos=down_gap.copy() + ) if signed: - # Stranded + signed: strand flip AND directional sign. Downstream (B - # after A) is positive by default but flips negative on A's '-' strand; - # upstream (B before A) is negative by default but flips positive on '-'. - downstream = _strand_flip_case( - strand_a, neg=exp.Neg(this=down_gap), pos=down_gap.copy() - ) + # Stranded + signed: upstream (B before A) is negative by default but + # flips positive on A's '-' strand (the directional sign layered on top + # of the strand flip). upstream = _strand_flip_case( strand_a, neg=up_gap, pos=exp.Neg(this=up_gap.copy()) ) else: - # Stranded but not signed: strand flip only. - downstream = _strand_flip_case( - strand_a, neg=exp.Neg(this=down_gap), pos=down_gap.copy() - ) + # Stranded but not signed: upstream carries the strand flip only. upstream = _strand_flip_case( strand_a, neg=exp.Neg(this=up_gap), pos=up_gap.copy() ) @@ -176,9 +231,21 @@ def _wrap_overlap_case( """Build the shared distance CASE skeleton common to all four branches. ``CASE WHEN chrom_a != chrom_b THEN NULL WHEN THEN 0 WHEN - end_a <= start_b THEN ELSE END`` — *downstream* is - the B-after-A gap, *upstream* the B-before-A gap (each already carrying any - sign / strand flip). + end_a <= start_b THEN ELSE END``. + + Parameters + ---------- + col_a, col_b : ResolvedColumn + The resolved A and B interval operands. + downstream : exp.Expression + The B-after-A gap expression (already carrying any sign / strand flip). + upstream : exp.Expression + The B-before-A gap expression (already carrying any sign / strand flip). + + Returns + ------- + exp.Expression + The assembled distance ``CASE`` expression. """ overlap = exp.and_( exp.LT(this=_frag(col_a.start), expression=_frag(col_b.end)), @@ -200,10 +267,24 @@ def _prepend_strand_guards( ) -> exp.Case: """Insert the strand-validity WHEN guards after the chrom guard. - Returns a CASE whose WHEN order is: chrom mismatch -> NULL, either strand - NULL -> NULL, strand_a is '.'/'?' -> NULL, strand_b is '.'/'?' -> NULL, - then the overlap/gap branches from *case*. Distance is undefined for an - unstranded ('.'/'?') or missing strand, matching the legacy emitter. + Distance is undefined for an unstranded (``'.'``/``'?'``) or missing strand, + matching the legacy emitter. + + Parameters + ---------- + case : exp.Case + The overlap/gap ``CASE`` to prepend the strand guards onto (mutated in + place). + strand_a, strand_b : str + The A and B strand-column SQL fragments. + + Returns + ------- + exp.Case + The same *case*, whose WHEN order is now: chrom mismatch -> NULL, either + strand NULL -> NULL, ``strand_a`` is ``'.'``/``'?'`` -> NULL, + ``strand_b`` is ``'.'``/``'?'`` -> NULL, then the original overlap/gap + branches. """ sa = _frag(strand_a) sb = _frag(strand_b) @@ -229,6 +310,13 @@ def _prepend_strand_guards( return case +# KEEP IN SYNC: this expander and +# ``BaseGIQLGenerator._generate_distance_case`` (base.py) build the *same* +# distance CASE by two routes. The legacy method is retained only because +# NEAREST still calls it for its ORDER BY / filter math; once NEAREST migrates +# to the expander path that method can be deleted and this duplication retired. +# Until then, any change to the distance math here must be mirrored there (and +# vice versa). The parity test in tests/test_distance_udf.py guards the drift. @register(GenericTarget, GIQLDistance) def expand_distance(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: """Expand a ``GIQLDistance`` node into a standard SQL ``CASE`` expression. @@ -236,8 +324,20 @@ def expand_distance(node: exp.Expression, ctx: ExpansionContext) -> exp.Expressi The portable expander registered for every target. Reads ``stranded`` / ``signed`` from the node and the pass-1 resolved interval operands, then builds the matching one of the four CASE shapes - (unsigned/signed x non-stranded/stranded). The result replaces the operator - node and is rendered by the active target's serializer. + (unsigned/signed x non-stranded/stranded). + + Parameters + ---------- + node : exp.Expression + The ``GIQLDistance`` operator node being expanded. + ctx : ExpansionContext + The expansion context carrying the node's pass-1 resolution. + + Returns + ------- + exp.Expression + The distance ``CASE`` that replaces the operator node and is rendered by + the active target's serializer. """ stranded = _bool_param(node.args.get("stranded")) signed = _bool_param(node.args.get("signed")) diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 1a4c949..a4be873 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -507,6 +507,17 @@ def _generate_distance_case( ) -> str: """Generate SQL CASE expression for distance calculation. + .. note:: + + KEEP IN SYNC: this method and the AST builder in + ``giql.expanders.distance`` (``expand_distance``) produce the *same* + distance CASE by two routes. DISTANCE itself migrated to the expander + (epic #137, issue #140); this method survives only because NEAREST + still calls it for its ORDER BY / filter math. Once NEAREST migrates, + delete this method and retire the duplication. Until then, any change + to the distance math here must be mirrored in the expander (and vice + versa); the parity test in tests/test_distance_udf.py guards drift. + Distances follow bedtools ``closest -d`` semantics: overlapping intervals report ``0``, book-ended (adjacent) intervals where ``A.end == B.start`` in half-open coordinates report ``1``, and a raw diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 1f806bc..81ee8e1 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -196,18 +196,19 @@ def transpile( with _reraise_as_value_error("Resolution error"): ast = resolve_operator_refs(ast, tables_container) - # Pass 2 of the normalization pipeline (epic #114): synthesize canonical - # __giql_canon_* wrapper CTEs for non-canonical interval operands of - # opted-in operators (GIQL_CANONICALIZE). No operator opts in yet, so this - # is a strict no-op until the per-operator port issues (#122, #123) land. + # Pass 2 of the normalization pipeline (epic #114): for each operator that + # opts into GIQL_CANONICALIZE, rewrite its non-canonical interval operands — + # synthesizing canonical __giql_canon_* wrapper CTEs — so downstream passes + # and emitters see canonical 0-based half-open coordinates. with _reraise_as_value_error("Canonicalization error"): ast = canonicalize_coordinates(ast) # Pass 3 of the normalization pipeline (epic #137): replace each opted-in # GIQL operator node with the AST its registered expander produces for the - # active target. No operator sets GIQL_EXPAND and the registry is empty, so - # this is a strict no-op until the per-operator migration issues land; the - # legacy *_sql emitters on the generator remain the fallback. + # active target. Each operator that opts in (GIQL_EXPAND) with a registered + # expander is rewritten here; any operator that is unflagged or has no + # registered expander falls through to its legacy *_sql emitter on the + # generator. expand_operators = ExpandOperators(target, tables_container) with _reraise_as_value_error("Expansion error"): ast = expand_operators.transform(ast) diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index e7f8c69..d83547d 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -1152,7 +1152,7 @@ def test_giqldistance_stranded_param_truthy_values_property( ): """ GIVEN a GIQLDistance with stranded parameter in various truthy representations - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN The parameter is parsed as True and strand-aware distance is calculated. """ sql = ( @@ -1175,7 +1175,7 @@ def test_giqldistance_stranded_param_falsy_values_property( ): """ GIVEN a GIQLDistance with stranded parameter in various falsy representations - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN The parameter is parsed as False and basic distance is calculated. """ sql = ( @@ -1197,7 +1197,7 @@ def test_giqldistance_signed_param_truthy_values_property( ): """ GIVEN a GIQLDistance with signed parameter in various truthy representations - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN The parameter is parsed as True and signed distance is calculated. """ sql = ( @@ -1219,7 +1219,7 @@ def test_giqldistance_signed_param_falsy_values_property( ): """ GIVEN a GIQLDistance with signed parameter in various falsy representations - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN The parameter is parsed as False and unsigned distance is calculated. """ sql = ( diff --git a/tests/test_distance_udf.py b/tests/test_distance_udf.py index bf12ef1..3def957 100644 --- a/tests/test_distance_udf.py +++ b/tests/test_distance_udf.py @@ -6,6 +6,10 @@ import duckdb import pytest +from hypothesis import given +from hypothesis import settings +from hypothesis import strategies as st +from sqlglot import exp from sqlglot import parse_one import giql # noqa: F401 (ensures the built-in expanders are registered) @@ -17,6 +21,10 @@ from giql.table import Tables from giql.targets import GenericTarget +#: This module executes generated SQL against a real in-memory DuckDB, so every +#: test here is an integration test (the marker is registered in pyproject). +pytestmark = pytest.mark.integration + def _generate(sql: str) -> str: """Parse, run normalization passes 1-3, then generate SQL. @@ -703,3 +711,184 @@ def test_stranded_signed_null_strand_returns_null(self): # Assert assert result is None, f"Expected NULL for '.' strand, got {result}" + + +# --- Drift guard: expand_distance vs the legacy _generate_distance_case ------- +# +# DISTANCE moved onto the AST-expansion pass (expand_distance), but +# BaseGIQLGenerator._generate_distance_case is retained because NEAREST still +# calls it. The two compute the same distance by different routes; these tests +# pin that they stay semantically equivalent until NEAREST migrates and the +# legacy method can be deleted. + +#: Column expressions both routes are evaluated over. ``a``/``b`` are the two +#: operand relations supplied by the parity harness's VALUES row. +_CHROM_A, _START_A, _END_A, _STRAND_A = 'a."chrom"', 'a."start"', 'a."end"', 'a."strand"' +_CHROM_B, _START_B, _END_B, _STRAND_B = 'b."chrom"', 'b."start"', 'b."end"', 'b."strand"' + +#: The four DISTANCE shapes: (id, stranded, signed). +_SHAPES = [ + ("unsigned_nonstranded", False, False), + ("signed_nonstranded", False, True), + ("unsigned_stranded", True, False), + ("signed_stranded", True, True), +] + +#: Rows exercised by the parity test: ordinary downstream/upstream gaps, +#: book-ended pairs, overlaps, a chrom mismatch, and strand-invalid ('.'/'?'/ +#: NULL) rows so every WHEN branch of both routes is covered. +_PARITY_ROWS = [ + ("chr1", 100, 200, "+", "chr1", 300, 400, "+"), # downstream gap + ("chr1", 300, 400, "+", "chr1", 100, 200, "+"), # upstream gap + ("chr1", 100, 200, "+", "chr1", 200, 300, "+"), # book-ended downstream + ("chr1", 200, 300, "+", "chr1", 100, 200, "+"), # book-ended upstream + ("chr1", 100, 200, "+", "chr1", 150, 250, "+"), # overlap + ("chr1", 100, 200, "-", "chr1", 300, 400, "+"), # '-' strand A, downstream + ("chr1", 300, 400, "-", "chr1", 100, 200, "+"), # '-' strand A, upstream + ("chr1", 100, 200, "+", "chr2", 300, 400, "+"), # chrom mismatch -> NULL + ("chr1", 100, 200, ".", "chr1", 300, 400, "+"), # strand A '.' -> NULL + ("chr1", 100, 200, "?", "chr1", 300, 400, "+"), # strand A '?' -> NULL + ("chr1", 100, 200, "+", "chr1", 300, 400, "."), # strand B '.' -> NULL + ("chr1", 100, 200, None, "chr1", 300, 400, "+"), # strand A NULL -> NULL +] + + +def _row_values_cte(row) -> str: + """Render one parity row as ``a``/``b`` relations via SELECT subqueries. + + *row* is ``(chrom_a, start_a, end_a, strand_a, chrom_b, ...)``; strands may + be ``None`` (rendered as SQL ``NULL``). + """ + ca, sa, ea, ta, cb, sb, eb, tb = row + + def _strand(value): + return "NULL" if value is None else f"'{value}'" + + a = ( + f"(SELECT '{ca}' AS chrom, {sa} AS start, {ea} AS \"end\", " + f"{_strand(ta)} AS strand) a" + ) + b = ( + f"(SELECT '{cb}' AS chrom, {sb} AS start, {eb} AS \"end\", " + f"{_strand(tb)} AS strand) b" + ) + return f"{a} CROSS JOIN {b}" + + +def _expander_distance_case(stranded: bool, signed: bool) -> str: + """Return the DISTANCE CASE the expander builds, isolated from its SELECT. + + Runs the real transpile passes over a DISTANCE query whose operands resolve + to ``a``/``b`` default columns, then lifts the generated CASE expression so + it can be re-embedded over arbitrary VALUES rows. + """ + args = ["a.interval", "b.interval"] + if stranded: + args.append("stranded := true") + if signed: + args.append("signed := true") + sql = ( + f"SELECT DISTANCE({', '.join(args)}) AS d " + "FROM (SELECT 'x' AS chrom, 0 AS start, 0 AS \"end\", '+' AS strand) a " + "CROSS JOIN (SELECT 'x' AS chrom, 0 AS start, 0 AS \"end\", '+' AS strand) b" + ) + tables = Tables() + ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables) + ast = canonicalize_coordinates(ast) + ast = ExpandOperators(GenericTarget(), tables).transform(ast) + # The single projected expression is the expander's CASE. + return ast.find(exp.Select).expressions[0].this.sql() + + +def _legacy_distance_case(stranded: bool, signed: bool) -> str: + """Return the CASE the legacy _generate_distance_case builds for ``a``/``b``.""" + return BaseGIQLGenerator()._generate_distance_case( + _CHROM_A, + _START_A, + _END_A, + _STRAND_A if stranded else None, + _CHROM_B, + _START_B, + _END_B, + _STRAND_B if stranded else None, + stranded=stranded, + signed=signed, + ) + + +def _eval_case(case_sql: str, row) -> object: + """Execute one distance CASE over one parity row, returning the scalar.""" + conn = duckdb.connect(":memory:") + try: + query = f"SELECT {case_sql} AS d FROM {_row_values_cte(row)}" + return conn.execute(query).fetchone()[0] + finally: + conn.close() + + +class TestDistanceExpanderLegacyParity: + """expand_distance and the retained _generate_distance_case agree row-for-row.""" + + @pytest.mark.parametrize( + "shape_id, stranded, signed", _SHAPES, ids=[s[0] for s in _SHAPES] + ) + def test_expander_matches_legacy_distance_case(self, shape_id, stranded, signed): + """ + GIVEN the four DISTANCE shapes (unsigned/signed x non-stranded/stranded) + plus overlap, chrom-mismatch, and strand-invalid input rows + WHEN the same inputs run through expand_distance and the retained + _generate_distance_case + THEN both routes return the identical scalar for every row, pinning the + two distance implementations against drift until NEAREST migrates. + """ + # Arrange + expander_case = _expander_distance_case(stranded, signed) + legacy_case = _legacy_distance_case(stranded, signed) + + # Act & assert + for row in _PARITY_ROWS: + expander_result = _eval_case(expander_case, row) + legacy_result = _eval_case(legacy_case, row) + assert expander_result == legacy_result, ( + f"{shape_id}: expander {expander_result!r} != " + f"legacy {legacy_result!r} for row {row}" + ) + + +class TestDistanceExpanderProperties: + """Property-based invariants of the expander's distance CASE.""" + + @settings(max_examples=200, deadline=None) + @given( + start_a=st.integers(min_value=0, max_value=10_000), + len_a=st.integers(min_value=1, max_value=5_000), + start_b=st.integers(min_value=0, max_value=10_000), + len_b=st.integers(min_value=1, max_value=5_000), + ) + def test_distance_invariants_hold(self, start_a, len_a, start_b, len_b): + """ + GIVEN random A and B intervals (start + positive length) + WHEN DISTANCE is evaluated unsigned, signed, and cross-chromosome + THEN unsigned == abs(signed), overlapping intervals report 0, and a + cross-chromosome pair reports NULL. + """ + # Arrange + end_a = start_a + len_a + end_b = start_b + len_b + same_chrom = ("chr1", start_a, end_a, "+", "chr1", start_b, end_b, "+") + cross_chrom = ("chr1", start_a, end_a, "+", "chr2", start_b, end_b, "+") + unsigned_case = _expander_distance_case(stranded=False, signed=False) + signed_case = _expander_distance_case(stranded=False, signed=True) + + # Act + unsigned = _eval_case(unsigned_case, same_chrom) + signed = _eval_case(signed_case, same_chrom) + cross = _eval_case(unsigned_case, cross_chrom) + + # Assert + assert unsigned == abs(signed) + overlaps = start_a < end_b and end_a > start_b + if overlaps: + assert unsigned == 0 + assert cross is None diff --git a/tests/test_expander.py b/tests/test_expander.py index a84077b..bb590e9 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -54,10 +54,12 @@ def _expander(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: #: The registry contents at import — the built-in expanders registered by -#: ``giql.expanders`` for already-migrated operators (DISJOIN as of #143). The -#: leak guards and ``clean_registry`` treat this as the baseline rather than an -#: empty registry, so the real built-in registrations survive isolating fixtures -#: and a leaking test is still caught against the true baseline. +#: ``giql.expanders`` for the already-migrated operators. The leak guards and +#: ``clean_registry`` treat this as the baseline rather than an empty registry, +#: so the real built-in registrations survive isolating fixtures and a leaking +#: test is still caught against the true baseline. +import giql.expanders # noqa: F401, E402 + _REGISTRY_BASELINE = REGISTRY.snapshot() @@ -483,7 +485,7 @@ def _expander(node, ctx): assert REGISTRY.resolve(DuckDBTarget(), GIQLDisjoin) is None assert (DuckDBTarget(), GIQLDisjoin) not in REGISTRY - def test_snapshot_is_independent_of_later_registrations(self): + def test_snapshot_should_not_observe_later_registrations(self): """Test that a snapshot does not observe registrations made after it. Given: @@ -506,7 +508,7 @@ def test_snapshot_is_independent_of_later_registrations(self): assert (DuckDBTarget(), GIQLDisjoin) in saved assert (GenericTarget(), Intersects) not in saved - def test_restore_replaces_entries_with_snapshot_contents(self): + def test_restore_should_replace_entries_with_snapshot_contents(self): """Test that restore returns the registry to a captured snapshot. Given: @@ -1019,13 +1021,13 @@ def test_transform_skips_unflagged_operator(self, clean_registry): """Test that an unflagged operator is left untouched even when registered. Given: - An expander registered for (GenericTarget, GIQLDisjoin) but the - operator's GIQL_EXPAND flag held off (DISJOIN ships it on, so the + An expander registered for a migrated operator but the operator's + GIQL_EXPAND flag held off (a migrated operator ships it on, so the control opts it out to isolate the per-type gate). When: Running the pass. Then: - The DISJOIN node should remain in the tree (gate requires both). + The operator node should remain in the tree (gate requires both). """ # Arrange clean_registry.register(GenericTarget(), GIQLDisjoin, _record("expanded")) @@ -1034,7 +1036,7 @@ def test_transform_skips_unflagged_operator(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_out(GIQLDisjoin): + with _opted_out(_A_MIGRATED_OPERATOR): result = pass_.transform(ast) # Assert @@ -1118,8 +1120,9 @@ def test_transpile_sql_unchanged_with_pass_inert(self): # The nine GIQL operator expression classes the ExpandOperators pass inspects. -# Every one must ship opted out (GIQL_EXPAND=False) at this step so the pass is a -# strict no-op until a later migration flips one alongside its expander. +# Each migrated operator ships opted in (GIQL_EXPAND=True) alongside its +# registered expander; the rest ship opted out (False) and fall through to the +# legacy emitter. from giql.expressions import Contains # noqa: E402 from giql.expressions import GIQLCluster # noqa: E402 from giql.expressions import GIQLDistance # noqa: E402 @@ -1140,9 +1143,9 @@ def test_transpile_sql_unchanged_with_pass_inert(self): ) #: Each operator's shipped GIQL_EXPAND default, captured from its own class dict -#: at import. A migrated operator (DISJOIN, #143) ships ``True``; the rest ship -#: ``False`` until their migrations land. The flag leak guard restores to these -#: shipped values rather than a blanket ``False``. +#: at import. A migrated operator ships ``True``; the rest ship ``False`` until +#: their migrations land. The flag leak guard restores to these shipped values +#: rather than a blanket ``False``. _SHIPPED_EXPAND_FLAGS = {op: op.__dict__.get("GIQL_EXPAND") for op in _OPERATOR_CLASSES} @@ -1150,6 +1153,9 @@ def test_transpile_sql_unchanged_with_pass_inert(self): _MIGRATED_OPERATORS = tuple( op for op in _OPERATOR_CLASSES if op.__dict__.get("GIQL_EXPAND") is True ) +assert _MIGRATED_OPERATORS, "expected at least one migrated operator" +#: An arbitrary migrated operator the operator-agnostic control tests target. +_A_MIGRATED_OPERATOR = _MIGRATED_OPERATORS[0] #: Operators not yet migrated — they ship GIQL_EXPAND=False. _UNMIGRATED_OPERATORS = tuple( op for op in _OPERATOR_CLASSES if op not in _MIGRATED_OPERATORS @@ -1759,13 +1765,13 @@ def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): """Test that only the flagged operator type is replaced when both registered. Given: - A DISJOIN and an INTERSECTS, both with registered expanders, but only - INTERSECTS flagged GIQL_EXPAND. + A migrated operator and an INTERSECTS, both with registered + expanders, but only INTERSECTS flagged GIQL_EXPAND. When: Running the pass. Then: - The INTERSECTS is replaced while the DISJOIN node remains (the gate is - per-type). + The INTERSECTS is replaced while the other operator node remains (the + gate is per-type). """ # Arrange clean_registry.register( @@ -1782,8 +1788,8 @@ def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): ) pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - # Act (DISJOIN ships flagged, so opt it out to hold it as the control) - with _opted_in(Intersects), _opted_out(GIQLDisjoin): + # Act (a migrated operator ships flagged, so opt it out to hold the control) + with _opted_in(Intersects), _opted_out(_A_MIGRATED_OPERATOR): result = pass_.transform(ast) # Assert @@ -2004,7 +2010,7 @@ class _opted_out: """Context manager opting an operator class out of GIQL_EXPAND for a test. The complement of :class:`_opted_in`: used by a control test that needs a - *migrated* operator (DISJOIN ships GIQL_EXPAND=True) to behave as if + *migrated* operator (one shipping GIQL_EXPAND=True) to behave as if unflagged, so the test can prove the pass gates per-type without the operator's shipped opt-in interfering. Restores the prior flag on exit. """ diff --git a/tests/test_transpile.py b/tests/test_transpile.py index ea770a8..5bf6138 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -458,8 +458,21 @@ def test_transpile_datafusion_accepted(self): "ANY('chr1:1000-2000', 'chr1:5000-6000')", ["peaks"], ), + ( + "SELECT DISTANCE(a.interval, b.interval) AS d " + "FROM peaks a, genes b", + ["peaks", "genes"], + ), + ], + ids=[ + "intersects_literal", + "contains", + "within", + "nearest", + "join", + "any", + "distance", ], - ids=["intersects_literal", "contains", "within", "nearest", "join", "any"], ) def test_transpile_datafusion_matches_generic_output(self, query, tables): """Test that datafusion is currently a pure alias for the generic target. @@ -483,6 +496,52 @@ def test_transpile_datafusion_matches_generic_output(self, query, tables): # Assert assert datafusion_sql == generic_sql + @pytest.mark.parametrize( + "query, tables", + [ + ( + "SELECT DISTANCE(a.interval, b.interval) AS d FROM peaks a, genes b", + ["peaks", "genes"], + ), + ( + "SELECT DISTANCE(a.interval, b.interval, stranded := true) AS d " + "FROM peaks a, genes b", + ["peaks", "genes"], + ), + ( + "SELECT DISTANCE(a.interval, b.interval, signed := true) AS d " + "FROM peaks a, genes b", + ["peaks", "genes"], + ), + ( + "SELECT DISTANCE(a.interval, b.interval, stranded := true, " + "signed := true) AS d FROM peaks a, genes b", + ["peaks", "genes"], + ), + ], + ids=["unsigned", "stranded", "signed", "stranded_signed"], + ) + def test_transpile_distance_is_byte_identical_across_targets(self, query, tables): + """Test that the migrated DISTANCE operator emits identical SQL everywhere. + + Given: + A DISTANCE query in each of the four shapes (unsigned/signed x + non-stranded/stranded), which migrated onto the AST-expansion pass + with a single generic expander. + When: + Transpiling it with dialect=None, "duckdb", and "datafusion". + Then: + The three outputs should be byte-identical — the generic expander + covers every target and DISTANCE has no engine-specific divergence. + """ + # Act + generic_sql = transpile(query, tables=tables, dialect=None) + duckdb_sql = transpile(query, tables=tables, dialect="duckdb") + datafusion_sql = transpile(query, tables=tables, dialect="datafusion") + + # Assert + assert generic_sql == duckdb_sql == datafusion_sql + def test_transpile_datafusion_accepts_intersects_bin_size(self): """Test that datafusion honours the binned-join bin size identically. From 6dda55a558dcbe7063f8c0f7139cb6ba115ec151 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 29 Jun 2026 11:05:42 -0400 Subject: [PATCH 093/142] test: Address round-2 review findings for DISTANCE Rewire the per-type opt-in control test to register, query, and opt out the same migrated operator via a branch-agnostic operator-query helper, with a paired positive so the gate is genuinely pinned. Refresh stale giqldistance_sql docstrings, reuse a DuckDB connection in the parity harness, tie the distance property test to the bedtools +1 offset, and document the bool() coercion. Soften the registry has_override and snapshot/restore docstrings, add a has_override truth-table test, guard the shipped-flag assumption, and make the inert-pass control operator-agnostic. --- src/giql/expander.py | 61 ++++--- src/giql/expanders/__init__.py | 10 ++ src/giql/expanders/distance.py | 12 +- tests/generators/test_base.py | 14 +- tests/test_distance_udf.py | 58 ++++-- tests/test_expander.py | 317 ++++++++++++++++++++++++++------- 6 files changed, 355 insertions(+), 117 deletions(-) diff --git a/src/giql/expander.py b/src/giql/expander.py index 3b5a900..ae9ca3e 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -227,13 +227,14 @@ def register( Notes ----- - Registering a *non-generic* ``(target, operator)`` expander where the - operator has a built-in whole-query join rewrite (notably - :class:`~giql.expressions.Intersects`, whose binned equi-join / DuckDB - IEJoin transformers run before expansion) signals that this expander - assumes responsibility for that rewrite: the built-in join transformers - are bypassed for that target so the operator node flows untouched into - :class:`ExpandOperators`. See :meth:`has_override`. + A *non-generic* ``(target, operator)`` entry is intended to also act as a + join-rewrite override for operators with a built-in whole-query join + rewrite (notably :class:`~giql.expressions.Intersects`, whose binned + equi-join / DuckDB IEJoin transformers run before expansion), letting a + per-target expander assume responsibility for that rewrite. That bypass + is intended for a future INTERSECTS consumer and is **not wired by any + caller yet** — no transformer consults :meth:`has_override` here (see + #141). """ self._expanders[(target, operator)] = _as_callable(expander) @@ -243,11 +244,13 @@ def resolve(self, target: Target, operator: type) -> ExpanderFn | None: Tries the exact ``(target, op)`` entry, then the ``(GenericTarget(), op)`` fallback, then ``None`` (legacy emitter). - A non-generic exact ``(target, op)`` entry is also a *join-rewrite - override* for operators with a built-in whole-query join rewrite (notably - :class:`~giql.expressions.Intersects`): registering one bypasses the - built-in binned / IEJoin transformers for that target (see - :meth:`register` and :meth:`has_override`). + A non-generic exact ``(target, op)`` entry is also intended to act as a + *join-rewrite override* for operators with a built-in whole-query join + rewrite (notably :class:`~giql.expressions.Intersects`). That override is + intended for a future INTERSECTS consumer and is **not wired by any + caller yet** — resolution does not itself bypass the built-in + binned / IEJoin transformers (see :meth:`register`, :meth:`has_override`, + and #141). """ fn = self._expanders.get((target, operator)) if fn is not None: @@ -259,15 +262,19 @@ def resolve(self, target: Target, operator: type) -> ExpanderFn | None: return None def has_override(self, target: Target, operator: type) -> bool: - """Whether an exact non-generic override supersedes built-in handling. + """Whether an exact non-generic ``(target, operator)`` entry is registered. Returns ``True`` only when *target* is not :class:`~giql.targets.GenericTarget` - and an exact ``(target, operator)`` entry is registered. Such an entry is a - target-specific override that supersedes built-in handling for that target - (e.g. it takes responsibility for the whole-query join rewrite that the - built-in transformers would otherwise perform); the portable + and an exact ``(target, operator)`` entry is registered; the portable ``(GenericTarget(), operator)`` fallback is *not* an override and does not count here. + + Such an entry is intended to mark a target-specific override that + supersedes built-in handling (e.g. taking responsibility for the + whole-query join rewrite the built-in transformers would otherwise + perform). That mechanism is intended for a future INTERSECTS consumer and + is **not wired by any caller yet** — no transformer consults this method + in the current pipeline (see #141). """ return target != GenericTarget() and (target, operator) in self._expanders @@ -294,12 +301,11 @@ def clear(self) -> None: def snapshot(self) -> dict[tuple[Target, type], ExpanderFn]: """Return a shallow copy of the current registrations. - The save half of a save/restore seam used primarily for test baseline - isolation (and which may serve a plugin that mutates the process-wide - :data:`REGISTRY` around a body): capture the baseline with this and hand - it back to :meth:`restore` afterward, so the built-in expanders - registered at import survive an isolating fixture that would otherwise - :meth:`clear` them permanently. It is not a committed plugin API. + The save half of a save/restore seam that supports test + baseline-isolation: capture the baseline with this and hand it back to + :meth:`restore` afterward, so the built-in expanders registered at import + survive an isolating fixture that would otherwise :meth:`clear` them + permanently. The returned dict is a fresh mapping (mutating it does not affect the registry), keyed by the same ``(target, operator)`` tuples. @@ -309,11 +315,10 @@ def snapshot(self) -> dict[tuple[Target, type], ExpanderFn]: def restore(self, snapshot: dict[tuple[Target, type], ExpanderFn]) -> None: """Replace all registrations with those captured by :meth:`snapshot`. - The restore half of the save/restore seam (test baseline isolation; may - also serve a plugin, but is not a committed plugin API). Drops every - current entry and re-installs exactly the *snapshot* contents, so a - fixture can return the registry to a previously captured baseline - regardless of what its body registered or cleared. + The restore half of the save/restore seam that supports test + baseline-isolation. Drops every current entry and re-installs exactly the + *snapshot* contents, so a fixture can return the registry to a previously + captured baseline regardless of what its body registered or cleared. """ self._expanders.clear() self._expanders.update(snapshot) diff --git a/src/giql/expanders/__init__.py b/src/giql/expanders/__init__.py index 042a07d..2628289 100644 --- a/src/giql/expanders/__init__.py +++ b/src/giql/expanders/__init__.py @@ -21,7 +21,17 @@ import importlib import pkgutil +from giql.expander import REGISTRY + for _module_info in pkgutil.iter_modules(__path__): if _module_info.name.startswith("_"): continue importlib.import_module(f"{__name__}.{_module_info.name}") + +# Fail loudly if discovery registered nothing. Under zipimport or PEP-420 +# namespace-package layouts ``pkgutil.iter_modules`` can yield no submodules, +# silently leaving the registry unpopulated; assert at least one expander landed +# so zero-discovery surfaces here rather than as a mystery legacy-path +# fallthrough. The check is branch-agnostic: it names no one operator, so it +# holds in every wave-3 worktree regardless of which expanders ship. +assert len(REGISTRY) > 0, "giql.expanders auto-discovery registered no expanders" diff --git a/src/giql/expanders/distance.py b/src/giql/expanders/distance.py index 530b56e..acd2c81 100644 --- a/src/giql/expanders/distance.py +++ b/src/giql/expanders/distance.py @@ -86,9 +86,15 @@ def _gap(minuend: str, subtrahend: str) -> exp.Expression: def _bool_param(param: exp.Expression | None) -> bool: """Coerce an optional DISTANCE boolean argument to a Python ``bool``. - Mirrors ``BaseGIQLGenerator._extract_bool_param`` so the expander reads the - ``stranded`` / ``signed`` keyword arguments identically to the legacy - emitter. + Mirrors ``BaseGIQLGenerator._extract_bool_param`` in how it reads the + ``stranded`` / ``signed`` keyword arguments, with one *intentional* + divergence: for an :class:`sqlglot.exp.Boolean` node this returns + ``bool(param.this)`` (a true Python ``bool``), whereas the legacy + ``_extract_bool_param`` returns the raw ``param_expr.this`` (already a + ``bool`` in practice, but unhardened). The coercion is strictly safer — it + guarantees a ``bool`` regardless of what the parser stored — and never + changes the observed branch selection, so the two stay behaviorally + equivalent. See TODO(#146): folding these two readers together. Parameters ---------- diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index d83547d..a7141c4 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -677,7 +677,7 @@ def test_giqlnearest_sql_parameter_handling_property( def test_giqldistance_sql_basic(self, tables_with_two_tables): """ GIVEN a GIQLDistance with two column references - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN CASE expression for distance calculation is generated. """ sql = ( @@ -700,7 +700,7 @@ def test_giqldistance_sql_basic(self, tables_with_two_tables): def test_giqldistance_sql_stranded(self, tables_with_two_tables): """ GIVEN a GIQLDistance with stranded := true - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN Strand-aware distance CASE expression is generated. """ sql = ( @@ -731,7 +731,7 @@ def test_giqldistance_sql_stranded(self, tables_with_two_tables): def test_giqldistance_sql_signed(self, tables_with_two_tables): """ GIVEN a GIQLDistance with signed := true - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN Signed distance CASE expression is generated. """ sql = ( @@ -754,7 +754,7 @@ def test_giqldistance_sql_signed(self, tables_with_two_tables): def test_giqldistance_sql_stranded_and_signed(self, tables_with_two_tables): """ GIVEN a GIQLDistance with both stranded and signed := true - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN Combined stranded+signed distance expression is generated. """ sql = ( @@ -791,7 +791,7 @@ def test_giqldistance_canonicalizes_closed_ends_apart_from_gap_parity( Given: Two 0-based closed-interval tables and DISTANCE. When: - giqldistance_sql is called. + the DISTANCE node is expanded. Then: It should canonicalize each table-side end as (end + 1) for the closed->half-open conversion, distinct from the bedtools-parity @@ -966,7 +966,7 @@ def test_giqlnearest_closed_interval_does_not_double_count_plus_one(self): def test_giqldistance_sql_literal_first_arg_error(self, tables_with_two_tables): """ GIVEN a GIQLDistance with literal range as first argument - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN ValueError is raised indicating literals not supported. """ sql = "SELECT DISTANCE('chr1:1000-2000', b.interval) as dist FROM features_b b" @@ -982,7 +982,7 @@ def test_giqldistance_sql_literal_first_arg_error(self, tables_with_two_tables): def test_giqldistance_sql_literal_second_arg_error(self, tables_with_two_tables): """ GIVEN a GIQLDistance with literal range as second argument - WHEN giqldistance_sql is called + WHEN the DISTANCE node is expanded THEN ValueError is raised indicating literals not supported. """ sql = "SELECT DISTANCE(a.interval, 'chr1:1000-2000') as dist FROM features_a a" diff --git a/tests/test_distance_udf.py b/tests/test_distance_udf.py index 3def957..a4a4351 100644 --- a/tests/test_distance_udf.py +++ b/tests/test_distance_udf.py @@ -817,12 +817,29 @@ def _legacy_distance_case(stranded: bool, signed: bool) -> str: ) -def _eval_case(case_sql: str, row) -> object: - """Execute one distance CASE over one parity row, returning the scalar.""" +def _eval_case(conn, case_sql: str, row) -> object: + """Execute one distance CASE over one parity row, returning the scalar. + + Reuses the caller-supplied *conn* (a module-scoped in-memory DuckDB) rather + than opening a fresh connection per row — every row is a standalone + ``SELECT ... FROM (VALUES)`` against no persistent state, so one connection + serves the whole parity sweep. + """ + query = f"SELECT {case_sql} AS d FROM {_row_values_cte(row)}" + return conn.execute(query).fetchone()[0] + + +@pytest.fixture(scope="module") +def parity_conn(): + """A module-scoped in-memory DuckDB connection for the parity/property sweep. + + Opened once and shared across every parity row and Hypothesis example + instead of reconnecting per row; each evaluation is a self-contained + ``SELECT`` over inline ``VALUES``, so no per-row isolation is needed. + """ conn = duckdb.connect(":memory:") try: - query = f"SELECT {case_sql} AS d FROM {_row_values_cte(row)}" - return conn.execute(query).fetchone()[0] + yield conn finally: conn.close() @@ -833,7 +850,9 @@ class TestDistanceExpanderLegacyParity: @pytest.mark.parametrize( "shape_id, stranded, signed", _SHAPES, ids=[s[0] for s in _SHAPES] ) - def test_expander_matches_legacy_distance_case(self, shape_id, stranded, signed): + def test_expander_matches_legacy_distance_case( + self, parity_conn, shape_id, stranded, signed + ): """ GIVEN the four DISTANCE shapes (unsigned/signed x non-stranded/stranded) plus overlap, chrom-mismatch, and strand-invalid input rows @@ -848,8 +867,8 @@ def test_expander_matches_legacy_distance_case(self, shape_id, stranded, signed) # Act & assert for row in _PARITY_ROWS: - expander_result = _eval_case(expander_case, row) - legacy_result = _eval_case(legacy_case, row) + expander_result = _eval_case(parity_conn, expander_case, row) + legacy_result = _eval_case(parity_conn, legacy_case, row) assert expander_result == legacy_result, ( f"{shape_id}: expander {expander_result!r} != " f"legacy {legacy_result!r} for row {row}" @@ -866,12 +885,16 @@ class TestDistanceExpanderProperties: start_b=st.integers(min_value=0, max_value=10_000), len_b=st.integers(min_value=1, max_value=5_000), ) - def test_distance_invariants_hold(self, start_a, len_a, start_b, len_b): + def test_distance_invariants_hold( + self, parity_conn, start_a, len_a, start_b, len_b + ): """ GIVEN random A and B intervals (start + positive length) WHEN DISTANCE is evaluated unsigned, signed, and cross-chromosome - THEN unsigned == abs(signed), overlapping intervals report 0, and a - cross-chromosome pair reports NULL. + THEN unsigned == abs(signed), overlapping intervals report 0, a + non-overlapping same-chrom pair reports the half-open gap + 1 + (bedtools parity ground truth), and a cross-chromosome pair reports + NULL. """ # Arrange end_a = start_a + len_a @@ -882,13 +905,22 @@ def test_distance_invariants_hold(self, start_a, len_a, start_b, len_b): signed_case = _expander_distance_case(stranded=False, signed=True) # Act - unsigned = _eval_case(unsigned_case, same_chrom) - signed = _eval_case(signed_case, same_chrom) - cross = _eval_case(unsigned_case, cross_chrom) + unsigned = _eval_case(parity_conn, unsigned_case, same_chrom) + signed = _eval_case(parity_conn, signed_case, same_chrom) + cross = _eval_case(parity_conn, unsigned_case, cross_chrom) # Assert assert unsigned == abs(signed) overlaps = start_a < end_b and end_a > start_b if overlaps: assert unsigned == 0 + else: + # Ground truth (bedtools closest -d): the unsigned distance of two + # non-overlapping same-chrom intervals is the raw half-open gap plus + # one. This ties the property test to the +1 offset directly, so + # dropping the +1 from the expander would fail here (not only in the + # legacy-parity test). The gap is end_a..start_b downstream or + # end_b..start_a upstream, whichever is non-negative. + gap_in_bases = max(start_b - end_a, start_a - end_b) + assert unsigned == gap_in_bases + 1 assert cross is None diff --git a/tests/test_expander.py b/tests/test_expander.py index bb590e9..196571b 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -104,7 +104,7 @@ def _expand_flag_leak_guard(): """Assert every operator's GIQL_EXPAND is restored at each test boundary. The symmetric partner of the registry leak guard: each operator class ships - a shipped GIQL_EXPAND default (``True`` for a migrated operator like DISJOIN, + a shipped GIQL_EXPAND default (``True`` for a migrated operator, ``False`` otherwise), and a test that flips one via ``_opted_in`` must restore it. A leaked flip would silently change the pass for a later test, so this catches anything that bypasses the exception-safe ``_opted_in`` manager by @@ -352,6 +352,55 @@ class _BadExpander: with pytest.raises(TypeError): registry.register(GenericTarget(), GIQLDisjoin, _BadExpander()) + def test_snapshot_should_not_observe_later_registrations(self): + """Test that a snapshot does not observe registrations made after it. + + Given: + A registry with one entry, captured by snapshot. + When: + A second entry is registered after the snapshot is taken. + Then: + The snapshot should still hold only the first entry (it is a copy, + not a live view). + """ + # Arrange + registry = ExpanderRegistry() + registry.register(DuckDBTarget(), GIQLDisjoin, _record("first")) + + # Act + saved = registry.snapshot() + registry.register(GenericTarget(), Intersects, _record("second")) + + # Assert + assert (DuckDBTarget(), GIQLDisjoin) in saved + assert (GenericTarget(), Intersects) not in saved + + def test_restore_should_replace_entries_with_snapshot_contents(self): + """Test that restore returns the registry to a captured snapshot. + + Given: + A snapshot of a registry with one entry, after which the registry is + cleared and a different entry registered. + When: + Restoring the snapshot. + Then: + The original entry should resolve again and the post-snapshot entry + should be gone. + """ + # Arrange + registry = ExpanderRegistry() + registry.register(DuckDBTarget(), GIQLDisjoin, _record("original")) + saved = registry.snapshot() + registry.clear() + registry.register(GenericTarget(), Intersects, _record("transient")) + + # Act + registry.restore(saved) + + # Assert + assert (DuckDBTarget(), GIQLDisjoin) in registry + assert (GenericTarget(), Intersects) not in registry + class TestExpanderRegistryFallbackGaps: """Edge cases of the registry fallback and op-scoped keying.""" @@ -485,54 +534,60 @@ def _expander(node, ctx): assert REGISTRY.resolve(DuckDBTarget(), GIQLDisjoin) is None assert (DuckDBTarget(), GIQLDisjoin) not in REGISTRY - def test_snapshot_should_not_observe_later_registrations(self): - """Test that a snapshot does not observe registrations made after it. + def test_has_override_should_return_true_when_exact_nongeneric_entry_registered( + self, + ): + """Test that has_override is True for an exact non-generic entry. Given: - A registry with one entry, captured by snapshot. + A registry with an exact (DuckDBTarget, op) entry registered. When: - A second entry is registered after the snapshot is taken. + Querying has_override for that exact key. Then: - The snapshot should still hold only the first entry (it is a copy, - not a live view). + It should return True (a non-generic exact entry is an override). """ # Arrange registry = ExpanderRegistry() - registry.register(DuckDBTarget(), GIQLDisjoin, _record("first")) - - # Act - saved = registry.snapshot() - registry.register(GenericTarget(), Intersects, _record("second")) + registry.register(DuckDBTarget(), GIQLDisjoin, _record("duckdb")) - # Assert - assert (DuckDBTarget(), GIQLDisjoin) in saved - assert (GenericTarget(), Intersects) not in saved + # Act & assert + assert registry.has_override(DuckDBTarget(), GIQLDisjoin) is True - def test_restore_should_replace_entries_with_snapshot_contents(self): - """Test that restore returns the registry to a captured snapshot. + def test_has_override_should_return_false_for_generic_fallback_entry(self): + """Test that has_override ignores the generic fallback entry. Given: - A snapshot of a registry with one entry, after which the registry is - cleared and a different entry registered. + A registry with only a (GenericTarget, op) entry registered. When: - Restoring the snapshot. + Querying has_override for a non-generic target's key. Then: - The original entry should resolve again and the post-snapshot entry - should be gone. + It should return False (the portable generic fallback is not an + override, even though resolve() would route to it). """ # Arrange registry = ExpanderRegistry() - registry.register(DuckDBTarget(), GIQLDisjoin, _record("original")) - saved = registry.snapshot() - registry.clear() - registry.register(GenericTarget(), Intersects, _record("transient")) + registry.register(GenericTarget(), GIQLDisjoin, _record("generic")) - # Act - registry.restore(saved) + # Act & assert + assert registry.has_override(DuckDBTarget(), GIQLDisjoin) is False + # And a generic target queried against its own entry is not an override. + assert registry.has_override(GenericTarget(), GIQLDisjoin) is False - # Assert - assert (DuckDBTarget(), GIQLDisjoin) in registry - assert (GenericTarget(), Intersects) not in registry + def test_has_override_should_return_false_when_unregistered(self): + """Test that has_override is False when nothing is registered. + + Given: + An empty registry. + When: + Querying has_override for any key. + Then: + It should return False (no entry, so no override). + """ + # Arrange + registry = ExpanderRegistry() + + # Act & assert + assert registry.has_override(DuckDBTarget(), GIQLDisjoin) is False class TestRegisterDecorator: @@ -1018,29 +1073,57 @@ def _expander(node, ctx): assert ctx.resolution.operator == "GIQLDisjoin" def test_transform_skips_unflagged_operator(self, clean_registry): - """Test that an unflagged operator is left untouched even when registered. + """Test that an opted-out operator is left untouched even when registered. Given: - An expander registered for a migrated operator but the operator's - GIQL_EXPAND flag held off (a migrated operator ships it on, so the - control opts it out to isolate the per-type gate). + A migrated operator with an expander registered for it, but its + GIQL_EXPAND flag opted out for the test — so the *same* operator is + registered, queried, and opted out, isolating the per-type gate. When: Running the pass. Then: - The operator node should remain in the tree (gate requires both). + The operator node should remain in the tree (the gate requires the + flag, so opting it out alone holds expansion off). """ # Arrange - clean_registry.register(GenericTarget(), GIQLDisjoin, _record("expanded")) - tables = _tables() - ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + operator = _A_MIGRATED_OPERATOR + clean_registry.register(GenericTarget(), operator, _record("expanded")) + ast, tables = _prepare_operator(operator) pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_out(_A_MIGRATED_OPERATOR): + with _opted_out(operator): result = pass_.transform(ast) # Assert - assert list(result.find_all(GIQLDisjoin)) + assert list(result.find_all(operator)) + + def test_transform_expands_flagged_operator(self, clean_registry): + """Test that the same operator expands once flagged (the paired positive). + + Given: + The same migrated operator and registered expander as the opt-out + control, but with its GIQL_EXPAND flag opted in. + When: + Running the pass. + Then: + The operator node should be replaced by the expander's output — so the + contrast with the opt-out control pins the per-type gate as + load-bearing, not vacuous. + """ + # Arrange + operator = _A_MIGRATED_OPERATOR + clean_registry.register(GenericTarget(), operator, _record("expanded")) + ast, tables = _prepare_operator(operator) + pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) + + # Act + with _opted_in(operator): + result = pass_.transform(ast) + + # Assert + assert not list(result.find_all(operator)) + assert result.find(exp.Literal).this == "expanded" def test_transform_skips_flagged_operator_with_no_expander(self, clean_registry): """Test that a flagged operator with no expander is left untouched. @@ -1093,30 +1176,34 @@ def test_transpile_sql_unchanged_with_pass_inert(self): """Test that transpile output is byte-identical for an unmigrated operator. Given: - A DISJOIN query (an operator not migrated onto the pass, so its - GIQL_EXPAND is False and no expander resolves), with the default - registry. + A CLUSTER query (an operator not migrated onto the pass in any wave-3 + branch, so its GIQL_EXPAND is False and no expander resolves), with + the default registry. When: - Transpiling with the wired-in pass versus a pass-bypassed reference - (its legacy emitter run directly). + Running the ExpandOperators pass (default REGISTRY) over the resolved + AST and serializing both the original and the pass-run AST. Then: - The SQL should match exactly and carry no expander alias prefix — the - pass is inert for any operator that has not been migrated. + The pass leaves the operator node in place, the serialized SQL is + byte-identical, and no expander alias prefix appears — the pass is + inert for any operator that has not been migrated. """ # Arrange - query = "SELECT * FROM DISJOIN(variants)" - tables = _tables() + query = "SELECT *, CLUSTER(interval) AS cluster_id FROM peaks" + tables = _tables(("peaks",)) ast = _prepare(query, tables) from giql.generators import BaseGIQLGenerator - expected = BaseGIQLGenerator(tables=tables).generate(ast) + before = BaseGIQLGenerator(tables=tables).generate(ast) + before_ops = len(list(ast.find_all(GIQLCluster))) - # Act - actual = transpile(query, tables=[Table("variants")]) + # Act — the wired-in pass over the default REGISTRY must be a no-op here. + result = expand_operators(ast, GenericTarget(), tables) + after = BaseGIQLGenerator(tables=tables).generate(result) # Assert - assert actual == expected - assert EXPAND_ALIAS_PREFIX not in actual + assert after == before + assert len(list(result.find_all(GIQLCluster))) == before_ops + assert EXPAND_ALIAS_PREFIX not in after # The nine GIQL operator expression classes the ExpandOperators pass inspects. @@ -1146,6 +1233,15 @@ def test_transpile_sql_unchanged_with_pass_inert(self): #: at import. A migrated operator ships ``True``; the rest ship ``False`` until #: their migrations land. The flag leak guard restores to these shipped values #: rather than a blanket ``False``. +# Pin the leak guard's assumption that every operator declares GIQL_EXPAND on its +# own class (so __dict__.get() reads the operator's shipped value rather than an +# inherited one). A future operator that omits its own flag would make the guard +# read ``None`` and silently lose its leak coverage; this one-time check closes +# that hole. +for _op in _OPERATOR_CLASSES: + assert "GIQL_EXPAND" in _op.__dict__, ( + f"{_op.__name__} must declare GIQL_EXPAND on its own class" + ) _SHIPPED_EXPAND_FLAGS = {op: op.__dict__.get("GIQL_EXPAND") for op in _OPERATOR_CLASSES} @@ -1160,6 +1256,62 @@ def test_transpile_sql_unchanged_with_pass_inert(self): _UNMIGRATED_OPERATORS = tuple( op for op in _OPERATOR_CLASSES if op not in _MIGRATED_OPERATORS ) +assert _UNMIGRATED_OPERATORS, "expected at least one unmigrated operator" + + +#: A minimal GIQL query producing one node of each operator class, keyed by the +#: class. Lets the control tests build a node for *any* operator — whichever the +#: branch ships migrated/unmigrated — so they stay operator-agnostic rather than +#: hard-wiring a particular operator. The second element is the table names the +#: query references (registered before pass 1). +_OPERATOR_QUERIES: dict[type, tuple[str, tuple[str, ...]]] = { + Intersects: ( + "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:1000-2000'", + ("variants",), + ), + Contains: ( + "SELECT * FROM variants WHERE interval CONTAINS 'chr1:1500-1600'", + ("variants",), + ), + Within: ( + "SELECT * FROM variants WHERE interval WITHIN 'chr1:1000-5000'", + ("variants",), + ), + SpatialSetPredicate: ( + "SELECT * FROM variants " + "WHERE interval INTERSECTS ANY('chr1:1000-2000', 'chr1:5000-6000')", + ("variants",), + ), + GIQLDistance: ( + "SELECT DISTANCE(a.interval, b.interval) AS d " + "FROM features_a a CROSS JOIN features_b b", + ("features_a", "features_b"), + ), + GIQLNearest: ( + "SELECT * FROM peaks CROSS JOIN LATERAL NEAREST(genes, k := 3)", + ("peaks", "genes"), + ), + GIQLDisjoin: ("SELECT * FROM DISJOIN(variants)", ("variants",)), + GIQLCluster: ( + "SELECT *, CLUSTER(interval) AS cluster_id FROM peaks", + ("peaks",), + ), + GIQLMerge: ("SELECT MERGE(interval) AS m FROM peaks", ("peaks",)), +} + + +def _prepare_operator(operator: type) -> tuple[exp.Expression, Tables]: + """Build a pass-1-resolved AST containing one node of *operator*. + + Operator-agnostic: looks the query up in :data:`_OPERATOR_QUERIES` so a + control test can exercise whichever operator a branch ships migrated or + unmigrated, rather than hard-wiring one operator class. Returns the resolved + AST and the Tables container it was resolved against (the same container must + be threaded into the pass). + """ + query, names = _OPERATOR_QUERIES[operator] + tables = _tables(names) + return _prepare(query, tables), tables class TestOperatorOptOut: @@ -1193,7 +1345,7 @@ def test_operator_class_ships_expand_enabled(self, operator): Given: A GIQL operator expression class migrated onto the ExpandOperators - pass (DISJOIN, #143). + pass. When: Reading its GIQL_EXPAND class attribute. Then: @@ -1207,6 +1359,32 @@ def test_operator_class_ships_expand_enabled(self, operator): assert flag is True +class TestMigratedOperatorsRegistered: + """Every migrated operator resolves a built-in expander in the process REGISTRY.""" + + @pytest.mark.parametrize( + "operator", _MIGRATED_OPERATORS, ids=lambda c: c.__name__ + ) + def test_migrated_operator_resolves_in_process_registry(self, operator): + """Test that each migrated operator resolves an expander in REGISTRY. + + Given: + A GIQL operator class that ships GIQL_EXPAND=True (migrated onto the + ExpandOperators pass). + When: + Resolving it against the import-populated process-wide REGISTRY for + the generic target. + Then: + A built-in expander should resolve — a migrated operator always has a + registered expander, so the pass never leaves it on a deleted emitter. + """ + # Arrange & act + resolved = REGISTRY.resolve(GenericTarget(), operator) + + # Assert + assert resolved is not None + + class TestOptedInRestoresFlag: """The _opted_in helper restores GIQL_EXPAND even when its body raises.""" @@ -1765,35 +1943,42 @@ def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): """Test that only the flagged operator type is replaced when both registered. Given: - A migrated operator and an INTERSECTS, both with registered - expanders, but only INTERSECTS flagged GIQL_EXPAND. + A genuinely-unmigrated operator (GIQLCluster, shipping + GIQL_EXPAND=False in every wave-3 branch) and an INTERSECTS, both with + registered expanders, but only INTERSECTS flagged GIQL_EXPAND for the + test. When: Running the pass. Then: - The INTERSECTS is replaced while the other operator node remains (the + The INTERSECTS is replaced while the unmigrated operator node remains + on its own shipped ``False`` flag — no opt-out ceremony needed (the gate is per-type). """ - # Arrange + # Arrange — the held-off subject is genuinely unmigrated: it survives on + # its own shipped GIQL_EXPAND=False, not on a test opt-out. + held_off = GIQLCluster + assert held_off in _UNMIGRATED_OPERATORS + assert held_off.GIQL_EXPAND is False clean_registry.register( - GenericTarget(), GIQLDisjoin, lambda n, c: exp.column("DJ") + GenericTarget(), held_off, lambda n, c: exp.column("CL") ) clean_registry.register( GenericTarget(), Intersects, lambda n, c: exp.column("IX") ) - tables = _tables(("variants", "peaks")) + tables = _tables(("peaks",)) ast = _prepare( - "SELECT * FROM DISJOIN(variants) " + "SELECT *, CLUSTER(interval) AS cluster_id FROM peaks " "WHERE EXISTS (SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1-100')", tables, ) pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - # Act (a migrated operator ships flagged, so opt it out to hold the control) - with _opted_in(Intersects), _opted_out(_A_MIGRATED_OPERATOR): + # Act — only INTERSECTS is opted in; the unmigrated operator stays off. + with _opted_in(Intersects): result = pass_.transform(ast) # Assert - assert list(result.find_all(GIQLDisjoin)) + assert list(result.find_all(held_off)) assert not list(result.find_all(Intersects)) def test_walk_shares_alias_sequence_across_sibling_expanders(self, clean_registry): From 55652630c17f7f5ee206df079e6148e6ceeba5ca Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 29 Jun 2026 12:49:53 -0400 Subject: [PATCH 094/142] refactor: Migrate INTERSECTS, CONTAINS, WITHIN, and set predicates to expanders Move INTERSECTS, CONTAINS, WITHIN, and the ANY/ALL set predicates off the legacy *_sql emitters onto the operator-expander registry (epic #137 wave 3). Each predicate expands to standard boolean AST built from the pass-1 resolved-column metadata, byte-identical to the deleted emitters. Capability-gate the binned and IEJoin join transformers on range_join_strategy and add a registry deferral (ExpanderRegistry.has_override) so a target-specific INTERSECTS override supersedes the built-in join rewrite, removing the IEJoin early-return's pipeline skip. Flip GIQL_EXPAND on the four predicate classes and delete their emitters and helpers. Squashed rebase onto main (post-#156) incorporating both review rounds: remove unreachable dispatch branches, add has_override with deferral-gate tests, guard intersects_bin_size under an override, relocate the direct expander tests to tests/expanders/test_intersects.py with CONTAINS/WITHIN column-join coverage, add error-message characterization, and reconcile the shared registry seam and docstrings. --- src/giql/expanders/intersects.py | 244 ++++++++++ src/giql/expressions.py | 24 +- src/giql/generators/base.py | 261 +---------- src/giql/transpile.py | 96 ++-- tests/expanders/__init__.py | 0 tests/expanders/test_intersects.py | 442 ++++++++++++++++++ tests/generators/test_base.py | 54 +-- .../datafusion/test_cross_target_oracle.py | 6 +- tests/test_expander.py | 55 +-- 9 files changed, 829 insertions(+), 353 deletions(-) create mode 100644 src/giql/expanders/intersects.py create mode 100644 tests/expanders/__init__.py create mode 100644 tests/expanders/test_intersects.py diff --git a/src/giql/expanders/intersects.py b/src/giql/expanders/intersects.py new file mode 100644 index 0000000..cbda7d6 --- /dev/null +++ b/src/giql/expanders/intersects.py @@ -0,0 +1,244 @@ +"""Generic expanders for the spatial predicates and set predicates (epic #137). + +Migrates INTERSECTS / CONTAINS / WITHIN and the ``ANY`` / ``ALL`` set predicates +off the legacy ``*_sql`` emitters on :class:`giql.generators.base.BaseGIQLGenerator` +and onto the operator-expander registry. Each expander turns one predicate node +into standard sqlglot AST built from the pass-1 :class:`~giql.resolver.ResolvedColumn` +metadata (already canonicalized to 0-based half-open by pass 2), so the emitted SQL +is byte-identical to the strings the legacy emitter produced. + +These are *node-local* predicate rewrites: an INTERSECTS / CONTAINS / WITHIN node +expands to a boolean ``(chrom = ... AND start < ... AND end > ...)`` expression that +replaces it in place. The whole-query column-to-column **join** rewrites (the binned +equi-join and the DuckDB IEJoin) remain capability-gated pre-pass transformers in +:mod:`giql.transformer` keyed on ``capabilities.range_join_strategy`` — they consume +a column-to-column INTERSECTS *join* before this pass runs, so by the time a +column-to-column INTERSECTS reaches an expander it is a residual predicate (e.g. +inside an ``OR``, or a join shape the transformer declined) that the legacy emitter +also rendered as a plain predicate. The expander handles that residual the same way. + +Only :class:`~giql.targets.GenericTarget` expanders are registered: spatial-predicate +*emission* is portable SQL-92 and does not vary by engine, so one generic expander +covers every target via the registry's ``(generic, op)`` fallback. +""" + +from __future__ import annotations + +from sqlglot import exp +from sqlglot import maybe_parse + +from giql.dialect import GIQLDialect +from giql.expander import ExpansionContext +from giql.expander import register +from giql.expressions import Contains +from giql.expressions import Intersects +from giql.expressions import SpatialSetPredicate +from giql.expressions import Within +from giql.range_parser import ParsedRange +from giql.range_parser import RangeParser +from giql.resolver import ResolvedColumn +from giql.targets import GenericTarget + + +def _fragment(fragment: str) -> exp.Expression: + """Parse a resolved SQL fragment (e.g. ``a."end"`` / ``'chr1'``) into AST. + + The pass-1 :class:`~giql.resolver.ResolvedColumn` carries column references as + pre-canonicalized SQL string fragments; parse them through the GIQL dialect so + the rebuilt predicate reserializes identically to the legacy emitter's string. + """ + parsed = maybe_parse(fragment, dialect=GIQLDialect) + if parsed is None: + # maybe_parse returns None only for an empty/None input; a ResolvedColumn + # fragment is never empty, so this is an internal invariant violation. + raise ValueError(f"Could not parse resolved column fragment: {fragment!r}") + return parsed + + +def _predicate_column(ctx: ExpansionContext, arg: str) -> ResolvedColumn: + """Return the :class:`ResolvedColumn` for predicate operand *arg*. + + Mirrors :meth:`giql.generators.base.BaseGIQLGenerator._predicate_operand`: the + expander consumes only the pass-1 resolution; a missing column means pass 1 did + not run (an internal invariant violation), so raise the historical message. + """ + resolution = ctx.resolution + if resolution is not None: + resolved = resolution.column(arg) + if resolved is not None: + return resolved + raise ValueError( + f"Spatial predicate operand {arg!r} was not resolved; run the " + "ResolveOperatorRefs pass (transpile pipeline) before generation." + ) + + +def _range_predicate( + column: ResolvedColumn, parsed: ParsedRange, op_type: str +) -> exp.Expression: + """Build the boolean AST for ``column ``. + + Reproduces :meth:`BaseGIQLGenerator._generate_range_predicate` as AST. The + column fragments are already canonical 0-based half-open (pass 2); the parsed + range is canonicalized by the caller. Returns a parenthesized boolean. + """ + chrom = _fragment(column.chrom) + start = _fragment(column.start) + end = _fragment(column.end) + chrom_lit = exp.Literal.string(parsed.chromosome) + r_start = exp.Literal.number(parsed.start) + r_end = exp.Literal.number(parsed.end) + + if op_type == "intersects": + # Ranges overlap if: start1 < end2 AND end1 > start2 + cond = exp.and_( + exp.EQ(this=chrom, expression=chrom_lit), + exp.LT(this=start, expression=r_end), + exp.GT(this=end, expression=r_start), + ) + elif op_type == "contains": + if parsed.end == parsed.start + 1: + # Point query: start1 <= point < end1 + cond = exp.and_( + exp.EQ(this=chrom, expression=chrom_lit), + exp.LTE(this=start, expression=r_start), + exp.GT(this=end, expression=r_start), + ) + else: + # Range query: start1 <= start2 AND end1 >= end2 + cond = exp.and_( + exp.EQ(this=chrom, expression=chrom_lit), + exp.LTE(this=start, expression=r_start), + exp.GTE(this=end, expression=r_end), + ) + elif op_type == "within": + # left within right: start1 >= start2 AND end1 <= end2 + cond = exp.and_( + exp.EQ(this=chrom, expression=chrom_lit), + exp.GTE(this=start, expression=r_start), + exp.LTE(this=end, expression=r_end), + ) + else: + raise ValueError(f"Unknown spatial op_type: {op_type!r}") + + return exp.paren(cond) + + +def _column_join( + left: ResolvedColumn, right: ResolvedColumn, op_type: str +) -> exp.Expression: + """Build the boolean AST for a column-to-column spatial predicate. + + Reproduces :meth:`BaseGIQLGenerator._generate_column_join` as AST. Both + operands' fragments are pre-canonicalized (pass 2). Returns a parenthesized + boolean. + """ + l_chrom, r_chrom = _fragment(left.chrom), _fragment(right.chrom) + l_start, r_start = _fragment(left.start), _fragment(right.start) + l_end, r_end = _fragment(left.end), _fragment(right.end) + + if op_type == "intersects": + cond = exp.and_( + exp.EQ(this=l_chrom, expression=r_chrom), + exp.LT(this=l_start, expression=r_end), + exp.GT(this=l_end, expression=r_start), + ) + elif op_type == "contains": + cond = exp.and_( + exp.EQ(this=l_chrom, expression=r_chrom), + exp.LTE(this=l_start, expression=r_start), + exp.GTE(this=l_end, expression=r_end), + ) + elif op_type == "within": + cond = exp.and_( + exp.EQ(this=l_chrom, expression=r_chrom), + exp.GTE(this=l_start, expression=r_start), + exp.LTE(this=l_end, expression=r_end), + ) + else: + raise ValueError(f"Unknown spatial op_type: {op_type!r}") + + return exp.paren(cond) + + +def _expand_spatial_op( + node: exp.Expression, ctx: ExpansionContext, op_type: str +) -> exp.Expression: + """Expand one INTERSECTS / CONTAINS / WITHIN node to a boolean predicate. + + Dispatches on the right operand exactly as the legacy emitter did: the + presence of a resolved right *column* — keyed off + ``ctx.resolution.column("expression")``, the slot pass 1 attaches a + :class:`ResolvedColumn` to when the right operand is a column reference — + selects the column-to-column path; its absence means the right operand is a + literal range, parsed in place. + """ + resolution = ctx.resolution + right_column = resolution.column("expression") if resolution is not None else None + left = _predicate_column(ctx, "this") + + if right_column is not None: + return _column_join(left, right_column, op_type) + + # Literal range string (e.g. interval INTERSECTS 'chr1:1000-2000'). Reproduce + # the legacy emitter's parse-and-wrap-error behavior verbatim: any parse + # failure (including the RangeParser's own ValueError) is wrapped in the + # historical "Could not parse genomic range" message. + right_expr = node.args.get("expression") + raw = right_expr.sql(dialect=GIQLDialect) if right_expr is not None else "" + try: + range_str = raw.strip("'\"") + parsed = RangeParser.parse(range_str).to_zero_based_half_open() + return _range_predicate(left, parsed, op_type) + except Exception as e: + raise ValueError(f"Could not parse genomic range: {raw}. Error: {e}") from e + + +@register(GenericTarget, Intersects) +def expand_intersects(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: + """Expand an INTERSECTS predicate to standard boolean SQL AST.""" + return _expand_spatial_op(node, ctx, "intersects") + + +@register(GenericTarget, Contains) +def expand_contains(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: + """Expand a CONTAINS predicate to standard boolean SQL AST.""" + return _expand_spatial_op(node, ctx, "contains") + + +@register(GenericTarget, Within) +def expand_within(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: + """Expand a WITHIN predicate to standard boolean SQL AST.""" + return _expand_spatial_op(node, ctx, "within") + + +@register(GenericTarget, SpatialSetPredicate) +def expand_spatial_set( + node: exp.Expression, ctx: ExpansionContext +) -> exp.Expression: + """Expand a quantified set predicate (``ANY`` / ``ALL``) to boolean SQL AST. + + Reproduces :meth:`BaseGIQLGenerator._generate_spatial_set`: the single left + column is compared against every literal range, and the per-range conditions + are OR-combined for ``ANY`` / AND-combined for ``ALL``, all wrapped in one + outer paren. + """ + operator = node.args["operator"] + quantifier = node.args["quantifier"] + ranges = node.args["ranges"] + + column = _predicate_column(ctx, "this") + op_type = operator.lower() + + conditions: list[exp.Expression] = [] + for range_expr in ranges: + range_str = range_expr.sql(dialect=GIQLDialect).strip("'\"") + parsed = RangeParser.parse(range_str).to_zero_based_half_open() + conditions.append(_range_predicate(column, parsed, op_type)) + + if quantifier.upper() == "ANY": + combined = exp.or_(*conditions) + else: + combined = exp.and_(*conditions) + + return exp.paren(combined) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 0c6bc80..4949fdd 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -114,8 +114,9 @@ class SpatialPredicate(exp.Binary): #: per-operator ``GIQL_EXPAND`` flag mirrors ``GIQL_CANONICALIZE``: an operator #: takes the new AST-expansion path only when it sets ``GIQL_EXPAND = True`` *and* #: an expander is registered for it; otherwise the legacy ``*_sql`` emitter runs. -#: Every operator defaults to ``False`` here, so the pass is a strict no-op until a -#: later migration step (#140+) flips one operator's flag alongside its expander. +#: This is the opt-out default: an operator inherits it and stays on the legacy +#: emitter until its migration step flips the flag to ``True`` alongside its +#: registered expander. Operators already migrated override it on their own class. _EXPAND = False @@ -126,7 +127,12 @@ class Intersects(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE - GIQL_EXPAND = _EXPAND + #: Migrated to the ExpandOperators registry (#141). A literal-range or + #: residual column-to-column INTERSECTS *predicate* expands through + #: ``giql.expanders.intersects``; a column-to-column INTERSECTS *join* is + #: consumed by the capability-gated binned / IEJoin pre-pass transformers + #: before this pass runs, so the predicate expander never sees it. + GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -141,7 +147,9 @@ class Contains(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE - GIQL_EXPAND = _EXPAND + #: Migrated to the ExpandOperators registry (#141); expands through + #: ``giql.expanders.intersects``. + GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -156,7 +164,9 @@ class Within(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE - GIQL_EXPAND = _EXPAND + #: Migrated to the ExpandOperators registry (#141); expands through + #: ``giql.expanders.intersects``. + GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -180,7 +190,9 @@ class SpatialSetPredicate(exp.Expression): } GIQL_CANONICALIZE = _CANONICALIZE - GIQL_EXPAND = _EXPAND + #: Migrated to the ExpandOperators registry (#141); expands through + #: ``giql.expanders.intersects``. + GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index a4be873..cc494dc 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -3,13 +3,8 @@ from giql.canonical import decanonical_end from giql.canonical import decanonical_start -from giql.expressions import Contains from giql.expressions import GIQLDisjoin from giql.expressions import GIQLNearest -from giql.expressions import Intersects -from giql.expressions import SpatialSetPredicate -from giql.expressions import Within -from giql.range_parser import ParsedRange from giql.range_parser import RangeParser from giql.resolver import META_KEY from giql.resolver import OperatorResolution @@ -41,45 +36,12 @@ def __init__(self, tables: Tables | None = None, **kwargs): super().__init__(**kwargs) self.tables = tables or Tables() - def intersects_sql(self, expression: Intersects) -> str: - """Generate standard SQL for INTERSECTS. - - :param expression: - INTERSECTS expression node - :return: - SQL predicate string - """ - return self._generate_spatial_op(expression, "intersects") - - def contains_sql(self, expression: Contains) -> str: - """Generate standard SQL for CONTAINS. - - :param expression: - CONTAINS expression node - :return: - SQL predicate string - """ - return self._generate_spatial_op(expression, "contains") - - def within_sql(self, expression: Within) -> str: - """Generate standard SQL for WITHIN. - - :param expression: - WITHIN expression node - :return: - SQL predicate string - """ - return self._generate_spatial_op(expression, "within") - - def spatialsetpredicate_sql(self, expression: SpatialSetPredicate) -> str: - """Generate SQL for spatial set predicates (ANY/ALL). - - :param expression: - SpatialSetPredicate expression node - :return: - SQL predicate string - """ - return self._generate_spatial_set(expression) + # INTERSECTS / CONTAINS / WITHIN and the ANY/ALL set predicates are migrated + # to the ExpandOperators registry (#141): they expand to standard boolean AST + # in ``giql.expanders.intersects`` before generation, so the generator no + # longer carries ``intersects_sql`` / ``contains_sql`` / ``within_sql`` / + # ``spatialsetpredicate_sql`` emitters or their ``_generate_spatial_*`` / + # ``_predicate_operand`` helpers. def giqlnearest_sql(self, expression: GIQLNearest) -> str: """Generate SQL for NEAREST function. @@ -601,217 +563,6 @@ def _generate_distance_case( f"ELSE ({start_a} - {end_b} + 1) END END" ) - def _predicate_operand(self, expression: exp.Expression, arg: str) -> ResolvedColumn: - """Return the :class:`ResolvedColumn` for a spatial predicate operand. - - Reads the column resolution attached to *expression* by the - ``ResolveOperatorRefs`` pass (pass 1). The emitter consumes only the - resolved metadata; all name/column resolution lives in the pass. - - :param expression: - The spatial predicate node carrying the resolution metadata. - :param arg: - The operand slot key (``"this"`` or ``"expression"``). - :return: - The resolved column metadata. - """ - resolution = expression.meta.get(META_KEY) - if isinstance(resolution, OperatorResolution): - resolved = resolution.column(arg) - if resolved is not None: - return resolved - - raise ValueError( - f"Spatial predicate operand {arg!r} was not resolved; run the " - "ResolveOperatorRefs pass (transpile pipeline) before generation." - ) - - def _generate_spatial_op(self, expression: exp.Binary, op_type: str) -> str: - """Generate SQL for a spatial operation. - - :param expression: - AST node (Intersects, Contains, or Within) - :param op_type: - 'intersects', 'contains', or 'within' - :return: - SQL predicate string - """ - right_raw = self.sql(expression, "expression") - - # Check if right side is a column reference or a literal range string - if "." in right_raw and not right_raw.startswith("'"): - # Column-to-column join (e.g., a.interval INTERSECTS b.interval) - left = self._predicate_operand(expression, "this") - right = self._predicate_operand(expression, "expression") - return self._generate_column_join(left, right, op_type) - else: - # Literal range string (e.g., interval INTERSECTS 'chr1:1000-2000') - try: - range_str = right_raw.strip("'\"") - parsed_range = RangeParser.parse(range_str).to_zero_based_half_open() - left = self._predicate_operand(expression, "this") - return self._generate_range_predicate(left, parsed_range, op_type) - except Exception as e: - raise ValueError( - f"Could not parse genomic range: {right_raw}. Error: {e}" - ) - - def _generate_range_predicate( - self, - column: ResolvedColumn, - parsed_range: ParsedRange, - op_type: str, - ) -> str: - """Generate SQL predicate for a range operation. - - :param column: - Resolved column operand (physical chrom/start/end fragments plus the - backing :class:`~giql.table.Table` config for canonicalization). - :param parsed_range: - Parsed genomic range - :param op_type: - 'intersects', 'contains', or 'within' - :return: - SQL predicate string - """ - # The alias-qualified column fragments come pre-resolved on the - # ResolvedColumn, already canonicalized to 0-based half-open by - # CanonicalizeCoordinates (pass 2, issue #123). The predicate returns a - # boolean, which is encoding-invariant, so no output de-canonicalization - # is needed. - chrom_col = column.chrom - start_col = column.start - end_col = column.end - - chrom = parsed_range.chromosome - start = parsed_range.start - end = parsed_range.end - - if op_type == "intersects": - # Ranges overlap if: start1 < end2 AND end1 > start2 - return ( - f"({chrom_col} = '{chrom}' " - f"AND {start_col} < {end} " - f"AND {end_col} > {start})" - ) - - elif op_type == "contains": - # Point query: start1 <= point < end1 - if end == start + 1: - return ( - f"({chrom_col} = '{chrom}' " - f"AND {start_col} <= {start} " - f"AND {end_col} > {start})" - ) - # Range query: start1 <= start2 AND end1 >= end2 - else: - return ( - f"({chrom_col} = '{chrom}' " - f"AND {start_col} <= {start} " - f"AND {end_col} >= {end})" - ) - - elif op_type == "within": - # Left within right: start1 >= start2 AND end1 <= end2 - return ( - f"({chrom_col} = '{chrom}' " - f"AND {start_col} >= {start} " - f"AND {end_col} <= {end})" - ) - - raise ValueError(f"Unknown operation: {op_type}") - - def _generate_column_join( - self, left: ResolvedColumn, right: ResolvedColumn, op_type: str - ) -> str: - """Generate SQL for column-to-column spatial joins. - - :param left: - Resolved left column operand (e.g., for 'a.interval'). - :param right: - Resolved right column operand (e.g., for 'b.interval'). - :param op_type: - 'intersects', 'contains', or 'within' - :return: - SQL predicate string - """ - # The alias-qualified chrom/start/end fragments come pre-resolved on the - # ResolvedColumns, already canonicalized to 0-based half-open by - # CanonicalizeCoordinates (pass 2, issue #123). The predicate returns a - # boolean (encoding-invariant), so no output de-canonicalization is needed. - l_chrom = left.chrom - r_chrom = right.chrom - l_start = left.start - l_end = left.end - r_start = right.start - r_end = right.end - - if op_type == "intersects": - # Ranges overlap if: chrom1 = chrom2 AND start1 < end2 AND end1 > start2 - return ( - f"({l_chrom} = {r_chrom} " - f"AND {l_start} < {r_end} " - f"AND {l_end} > {r_start})" - ) - - elif op_type == "contains": - # Left contains right: chrom1 = chrom2 AND start1 <= start2 AND end1 >= end2 - return ( - f"({l_chrom} = {r_chrom} " - f"AND {l_start} <= {r_start} " - f"AND {l_end} >= {r_end})" - ) - - elif op_type == "within": - # Left within right: chrom1 = chrom2 AND start1 >= start2 AND end1 <= end2 - return ( - f"({l_chrom} = {r_chrom} " - f"AND {l_start} >= {r_start} " - f"AND {l_end} <= {r_end})" - ) - - raise ValueError(f"Unknown operation: {op_type}") - - def _generate_spatial_set(self, expression: SpatialSetPredicate) -> str: - """Generate SQL for spatial set predicates (ANY/ALL). - - Examples: - column INTERSECTS ANY(...) -> (condition1 OR condition2 OR ...) - column INTERSECTS ALL(...) -> (condition1 AND condition2 AND ...) - - :param expression: - SpatialSetPredicate expression node - :return: - SQL predicate string - """ - operator = expression.args["operator"] - quantifier = expression.args["quantifier"] - ranges = expression.args["ranges"] - - # Resolve the (single) left column operand once; every range condition - # compares against the same column. The set predicate's ranges are - # always literals, so only this operand needs resolution. - column = self._predicate_operand(expression, "this") - - # Parse all ranges - parsed_ranges = [] - for range_expr in ranges: - range_str = self.sql(range_expr).strip("'\"") - parsed_range = RangeParser.parse(range_str).to_zero_based_half_open() - parsed_ranges.append(parsed_range) - - op_type = operator.lower() - - # Generate conditions for each range - conditions = [] - for parsed_range in parsed_ranges: - condition = self._generate_range_predicate(column, parsed_range, op_type) - conditions.append(condition) - - # Combine with AND (for ALL) or OR (for ANY) - combinator = " OR " if quantifier.upper() == "ANY" else " AND " - return "(" + combinator.join(conditions) + ")" - def _detect_nearest_mode( self, expression: GIQLNearest, parent_expression: exp.Expression | None = None ) -> str: diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 81ee8e1..b5568e2 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -14,7 +14,9 @@ import giql.expanders # noqa: F401 from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect +from giql.expander import REGISTRY from giql.expander import ExpandOperators +from giql.expressions import Intersects from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Table @@ -151,41 +153,78 @@ def transpile( "of the binned equi-join. Pass one or the other, not both." ) + # The INTERSECTS join-rewrite override (governs the three uses below). + # + # A *target-specific* ``(target, Intersects)`` registry entry — the public + # extension hook, matched by ``ExpanderRegistry.has_override`` — takes over the + # INTERSECTS join rewrite entirely. ``has_override`` deliberately matches only + # an *exact non-generic* entry: the built-in ``(GenericTarget(), Intersects)`` + # predicate expander is NOT a join-strategy override (it only renders the + # residual / literal-range predicates the join transformers leave behind), so + # it must not disable the join rewrite. When an override is present, all three + # built-in join paths below are bypassed for that target so the INTERSECTS node + # flows untouched into ExpandOperators, which dispatches it to that expander: + # 1. ``intersects_bin_size`` is rejected — it only configures the built-in + # binned transformer the override supersedes (rejected here, parallel to + # the iejoin rejection above, rather than silently dropped); + # 2. the DuckDB IEJoin short-circuit is skipped (the registry-deferral the + # IEJoin early-return used to preclude, #141); + # 3. the binned-join transformer is skipped. + target_overrides_intersects = REGISTRY.has_override(target, Intersects) + if target_overrides_intersects and intersects_bin_size is not None: + raise ValueError( + "intersects_bin_size has no effect when a target-specific " + f"(target={target.name!r}, Intersects) expander is registered; that " + "expander supersedes the built-in binned join transformer the bin " + "size configures. Pass one or the other, not both." + ) + tables_container = _build_tables(tables) with _reraise_as_value_error("Parse error", query=giql): ast = parse_one(giql, dialect=GIQLDialect) + # The column-to-column INTERSECTS *join* rewrites are capability-gated + # pre-pass transformers (epic #137, #141): the target's + # ``range_join_strategy`` selects the DuckDB IEJoin plan or the generic + # binned equi-join. They run on the raw parsed AST (before resolution, which + # rewrites the genomic column name) and consume a column-to-column INTERSECTS + # *join* so it never reaches the predicate expander; a literal-range or + # residual column-to-column INTERSECTS *predicate* survives to pass 3. + # ``target_overrides_intersects`` (computed above) gates whether these + # built-in join paths run — see its definition for the override rationale. + # Falls back to the binned plan for unsupported shapes — see # IntersectsDuckDBIEJoinTransformer.transform_to_sql for the complete - # fallback set. - if uses_iejoin: + # fallback set. The IEJoin transformer emits a whole-query string, so when it + # produces output it must short-circuit the AST pipeline. This never skips an + # INTERSECTS that pass 3 would expand: ``_classify_extras`` forces the binned + # fallback (returning None here) for any query carrying a residual INTERSECTS + # alongside the join, so a query the IEJoin transformer accepts has no residual + # INTERSECTS left for the expander. (A residual CONTAINS/WITHIN/ANY beside an + # IEJoin is a pre-existing IEJoin limitation that errors identically on main.) + if uses_iejoin and not target_overrides_intersects: duckdb_transformer = IntersectsDuckDBIEJoinTransformer(tables_container) with _reraise_as_value_error("Transformation error"): duckdb_sql = duckdb_transformer.transform_to_sql(ast) if duckdb_sql is not None: - # WARNING: this early return emits the legacy IEJoin SQL directly and - # SKIPS the normalization pipeline below — pass 1 (resolution), pass 2 - # (canonicalization), and pass 3 (ExpandOperators, constructed ~40 - # lines down). The ExpandOperators registry is therefore NOT consulted - # on this path: a flagged operator on an IEJoin-eligible duckdb query - # is left un-expanded. This is benign today (the registry is empty and - # no operator opts in), but any DuckDB-pathed operator migration (#141) - # must either run expansion BEFORE this early return or have the IEJoin - # transformer defer to the registry. See the strict-xfail - # characterization test pinning this gap in tests/test_expander.py. return duckdb_sql - intersects_transformer = IntersectsBinnedJoinTransformer( - tables_container, - bin_size=intersects_bin_size, - ) merge_transformer = MergeTransformer(tables_container) cluster_transformer = ClusterTransformer(tables_container) generator = BaseGIQLGenerator(tables=tables_container) with _reraise_as_value_error("Transformation error"): - ast = intersects_transformer.transform(ast) + # Reaching here with an iejoin target means the IEJoin transformer + # declined the query (returned None) and fell back to the binned plan, + # exactly as before. ``intersects_bin_size`` is rejected up front for + # iejoin targets, so the binned transformer always sees its default there. + if not target_overrides_intersects: + intersects_transformer = IntersectsBinnedJoinTransformer( + tables_container, + bin_size=intersects_bin_size, + ) + ast = intersects_transformer.transform(ast) ast = merge_transformer.transform(ast) ast = cluster_transformer.transform(ast) @@ -196,22 +235,21 @@ def transpile( with _reraise_as_value_error("Resolution error"): ast = resolve_operator_refs(ast, tables_container) - # Pass 2 of the normalization pipeline (epic #114): for each operator that - # opts into GIQL_CANONICALIZE, rewrite its non-canonical interval operands — - # synthesizing canonical __giql_canon_* wrapper CTEs — so downstream passes - # and emitters see canonical 0-based half-open coordinates. + # Pass 2 of the normalization pipeline (epic #114): synthesize canonical + # __giql_canon_* wrapper CTEs for non-canonical interval operands of operators + # that opt in via GIQL_CANONICALIZE; those operators are rewritten here, and + # operators that do not opt in are left untouched. with _reraise_as_value_error("Canonicalization error"): ast = canonicalize_coordinates(ast) - # Pass 3 of the normalization pipeline (epic #137): replace each opted-in - # GIQL operator node with the AST its registered expander produces for the - # active target. Each operator that opts in (GIQL_EXPAND) with a registered - # expander is rewritten here; any operator that is unflagged or has no - # registered expander falls through to its legacy *_sql emitter on the - # generator. - expand_operators = ExpandOperators(target, tables_container) + # Pass 3 of the normalization pipeline (epic #137): replace each GIQL operator + # node that opts in (GIQL_EXPAND) and resolves a registered expander with the + # AST that expander produces for the active target. Operators that are + # unflagged or resolve no expander are left untouched and the generator renders + # them via their legacy ``*_sql`` emitter as before. + expand_pass = ExpandOperators(target, tables_container) with _reraise_as_value_error("Expansion error"): - ast = expand_operators.transform(ast) + ast = expand_pass.transform(ast) with _reraise_as_value_error("Transpilation error"): sql = generator.generate(ast) diff --git a/tests/expanders/__init__.py b/tests/expanders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/expanders/test_intersects.py b/tests/expanders/test_intersects.py new file mode 100644 index 0000000..aff4e86 --- /dev/null +++ b/tests/expanders/test_intersects.py @@ -0,0 +1,442 @@ +"""Direct unit tests for the spatial / set predicate expanders (#141). + +These call ``expand_intersects`` / ``expand_contains`` / ``expand_within`` / +``expand_spatial_set`` directly with a hand-built :class:`ExpansionContext`, +characterizing each dispatch branch (column-to-column vs literal range; CONTAINS +point vs range; ANY/OR vs ALL/AND) and pinning the chosen error messages on +invalid input. They sit outside ``tests/test_expander.py`` so they do not touch +that file's shared, operator-agnostic fixture/infra region. +""" + +import pytest +from sqlglot import exp +from sqlglot import parse_one + +from giql.dialect import GIQLDialect +from giql.expander import REGISTRY +from giql.expander import ExpansionContext +from giql.expanders.intersects import expand_contains +from giql.expanders.intersects import expand_intersects +from giql.expanders.intersects import expand_spatial_set +from giql.expanders.intersects import expand_within +from giql.expressions import Contains +from giql.expressions import Intersects +from giql.expressions import SpatialSetPredicate +from giql.expressions import Within +from giql.resolver import OperatorResolution +from giql.resolver import ResolvedColumn +from giql.table import Tables +from giql.targets import DataFusionTarget +from giql.targets import GenericTarget +from giql.transpile import transpile + +_LEFT = ResolvedColumn( + chrom='a."chrom"', start='a."start"', end='a."end"', strand=None, table=None +) +_RIGHT = ResolvedColumn( + chrom='b."chrom"', start='b."start"', end='b."end"', strand=None, table=None +) +_OPERATOR_TYPES = (Intersects, Contains, Within, SpatialSetPredicate) + + +def _context(query: str, columns: dict[str, ResolvedColumn]) -> tuple: + """Find the spatial operator in *query* and build a context with *columns*.""" + root = parse_one(query, dialect=GIQLDialect) + node = next(n for n in root.walk() if isinstance(n, _OPERATOR_TYPES)) + resolution = OperatorResolution( + operator=type(node).__name__, slots={}, columns=columns + ) + ctx = ExpansionContext(node, resolution, GenericTarget(), Tables()) + return node, ctx + + +def _sql(expression: exp.Expression) -> str: + """Serialize a built expression through the GIQL dialect.""" + return expression.sql(dialect=GIQLDialect) + + +class TestSpatialExpanders: + """Direct expansion of the spatial / set predicate expanders (#141).""" + + def test_expand_intersects_should_build_overlap_predicate_when_literal_range(self): + """Test that a literal-range INTERSECTS expands to the overlap predicate. + + Given: + An INTERSECTS node whose right operand is a literal range and a + context resolving only the left column. + When: + Expanding it. + Then: + It should build the overlap boolean (chrom = lit AND start < end2 AND + end > start2) with the right operand as numeric literals. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval INTERSECTS 'chr1:1000-2000'", + {"this": _LEFT}, + ) + + # Act + result = expand_intersects(node, ctx) + + # Assert + assert _sql(result) == ( + '(a."chrom" = \'chr1\' AND a."start" < 2000 AND a."end" > 1000)' + ) + + def test_expand_intersects_should_build_join_predicate_when_column_to_column(self): + """Test that a column-to-column INTERSECTS expands to a join predicate. + + Given: + An INTERSECTS node with a resolved right *column* (the dispatch keys on + ctx.resolution.column("expression")). + When: + Expanding it. + Then: + It should compare the two columns' endpoints rather than literals. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a JOIN b ON a.interval INTERSECTS b.interval", + {"this": _LEFT, "expression": _RIGHT}, + ) + + # Act + result = expand_intersects(node, ctx) + + # Assert + assert _sql(result) == ( + '(a."chrom" = b."chrom" AND a."start" < b."end" AND a."end" > b."start")' + ) + + def test_expand_contains_should_build_point_predicate_when_single_base(self): + """Test that a single-base CONTAINS expands to the point-containment form. + + Given: + A CONTAINS node whose literal range is a single base (end == start+1). + When: + Expanding it. + Then: + It should use the point form (start <= point AND end > point), not the + range form. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval CONTAINS 'chr1:1000'", {"this": _LEFT} + ) + + # Act + result = expand_contains(node, ctx) + + # Assert + assert _sql(result) == ( + '(a."chrom" = \'chr1\' AND a."start" <= 1000 AND a."end" > 1000)' + ) + + def test_expand_contains_should_build_range_predicate_when_multi_base(self): + """Test that a multi-base CONTAINS expands to the range-containment form. + + Given: + A CONTAINS node whose literal range spans more than one base. + When: + Expanding it. + Then: + It should use the range form (start <= start2 AND end >= end2). + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval CONTAINS 'chr1:1000-2000'", + {"this": _LEFT}, + ) + + # Act + result = expand_contains(node, ctx) + + # Assert + assert _sql(result) == ( + '(a."chrom" = \'chr1\' AND a."start" <= 1000 AND a."end" >= 2000)' + ) + + def test_expand_contains_should_build_join_predicate_when_column_to_column(self): + """Test that a column-to-column CONTAINS expands to the containment join. + + Given: + A CONTAINS node with a resolved right *column* (the dispatch keys on + ctx.resolution.column("expression")). + When: + Expanding it. + Then: + It should build the left-contains-right predicate (start1 <= start2 + AND end1 >= end2) comparing the two columns' endpoints. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a JOIN b ON a.interval CONTAINS b.interval", + {"this": _LEFT, "expression": _RIGHT}, + ) + + # Act + result = expand_contains(node, ctx) + + # Assert + assert _sql(result) == ( + '(a."chrom" = b."chrom" AND a."start" <= b."start" AND a."end" >= b."end")' + ) + + def test_expand_within_should_build_containment_predicate_when_literal_range(self): + """Test that WITHIN expands to the left-within-right containment form. + + Given: + A WITHIN node with a literal range. + When: + Expanding it. + Then: + It should build start >= start2 AND end <= end2. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval WITHIN 'chr1:1000-2000'", {"this": _LEFT} + ) + + # Act + result = expand_within(node, ctx) + + # Assert + assert _sql(result) == ( + '(a."chrom" = \'chr1\' AND a."start" >= 1000 AND a."end" <= 2000)' + ) + + def test_expand_within_should_build_join_predicate_when_column_to_column(self): + """Test that a column-to-column WITHIN expands to the within join predicate. + + Given: + A WITHIN node with a resolved right *column* (the dispatch keys on + ctx.resolution.column("expression")). + When: + Expanding it. + Then: + It should build the left-within-right predicate (start1 >= start2 AND + end1 <= end2) comparing the two columns' endpoints. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a JOIN b ON a.interval WITHIN b.interval", + {"this": _LEFT, "expression": _RIGHT}, + ) + + # Act + result = expand_within(node, ctx) + + # Assert + assert _sql(result) == ( + '(a."chrom" = b."chrom" AND a."start" >= b."start" AND a."end" <= b."end")' + ) + + def test_expand_spatial_set_should_or_combine_conditions_when_any(self): + """Test that an ANY set predicate OR-combines its per-range conditions. + + Given: + An INTERSECTS ANY node over two literal ranges. + When: + Expanding it. + Then: + The two per-range overlap predicates should be OR-combined inside one + outer paren. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval " + "INTERSECTS ANY ('chr1:1-100', 'chr1:200-300')", + {"this": _LEFT}, + ) + + # Act + result = expand_spatial_set(node, ctx) + + # Assert + assert _sql(result) == ( + '((a."chrom" = \'chr1\' AND a."start" < 100 AND a."end" > 1) OR ' + '(a."chrom" = \'chr1\' AND a."start" < 300 AND a."end" > 200))' + ) + + def test_expand_spatial_set_should_and_combine_conditions_when_all(self): + """Test that an ALL set predicate AND-combines its per-range conditions. + + Given: + An INTERSECTS ALL node over two literal ranges. + When: + Expanding it. + Then: + The two per-range overlap predicates should be AND-combined inside one + outer paren. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval " + "INTERSECTS ALL ('chr1:1-100', 'chr1:200-300')", + {"this": _LEFT}, + ) + + # Act + result = expand_spatial_set(node, ctx) + + # Assert + assert _sql(result) == ( + '((a."chrom" = \'chr1\' AND a."start" < 100 AND a."end" > 1) AND ' + '(a."chrom" = \'chr1\' AND a."start" < 300 AND a."end" > 200))' + ) + + +@pytest.fixture +def isolated_registry(): + """Snapshot/restore the process REGISTRY so a test can register an override.""" + saved = REGISTRY.snapshot() + try: + yield REGISTRY + finally: + REGISTRY.restore(saved) + + +class TestBinnedTargetOverrideDeferral: + """A target-specific Intersects override defers the binned join rewrite (#141).""" + + def test_transpile_should_skip_binned_rewrite_when_target_override( + self, isolated_registry + ): + """Test that a (target, Intersects) override bypasses the binned transformer. + + Given: + A column-to-column INTERSECTS join on the generic binned path + (dialect='datafusion') with a (DataFusionTarget, Intersects) override + registered. + When: + Transpiling. + Then: + The override's sentinel reaches the SQL and no binned equi-join + artifact is emitted — the override takes over the join rewrite that the + built-in binned transformer would otherwise perform. + """ + # Arrange + isolated_registry.register( + DataFusionTarget(), + Intersects, + lambda n, c: exp.column("BINNED_OVERRIDE_SENTINEL"), + ) + query = ( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act + sql = transpile(query, tables=["peaks", "genes"], dialect="datafusion") + + # Assert + assert "BINNED_OVERRIDE_SENTINEL" in sql + assert "_bins" not in sql + + def test_transpile_should_reject_bin_size_when_target_override( + self, isolated_registry + ): + """Test that bin size is rejected under a binned-target Intersects override. + + Given: + A (DataFusionTarget, Intersects) override registered. + When: + Transpiling with intersects_bin_size set (which only configures the + built-in binned transformer the override supersedes). + Then: + transpile() raises ValueError rather than silently dropping the bin + size, parallel to the iejoin rejection. + """ + # Arrange + isolated_registry.register( + DataFusionTarget(), + Intersects, + lambda n, c: exp.column("BINNED_OVERRIDE_SENTINEL"), + ) + query = ( + "SELECT a.start FROM peaks a " + "JOIN genes b ON a.interval INTERSECTS b.interval" + ) + + # Act & assert + with pytest.raises(ValueError, match=r"intersects_bin_size has no effect"): + transpile( + query, + tables=["peaks", "genes"], + dialect="datafusion", + intersects_bin_size=5000, + ) + + +class TestSpatialExpanderErrors: + """Characterization tests pinning the chosen error messages on invalid input.""" + + def test_expand_intersects_should_wrap_parse_error_when_invalid_range(self): + """Test that an unparseable literal range raises the wrapped diagnostic. + + Given: + An INTERSECTS node whose literal range string cannot be parsed. + When: + Expanding it. + Then: + It should raise ValueError with the historical "Could not parse + genomic range" wrapper, chained from the underlying parse error. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval INTERSECTS 'invalid'", {"this": _LEFT} + ) + + # Act & assert + with pytest.raises(ValueError, match=r"Could not parse genomic range") as exc: + expand_intersects(node, ctx) + assert exc.value.__cause__ is not None + + def test_expand_intersects_should_raise_invariant_when_left_unresolved(self): + """Test that a missing left-operand resolution raises the invariant error. + + Given: + An INTERSECTS node whose context resolved no "this" column (pass 1 did + not run). + When: + Expanding it. + Then: + It should raise ValueError naming the unresolved operand and pointing + at the ResolveOperatorRefs pass. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval INTERSECTS 'chr1:1-100'", {} + ) + + # Act & assert + with pytest.raises( + ValueError, match=r"Spatial predicate operand 'this' was not resolved" + ): + expand_intersects(node, ctx) + + def test_expand_spatial_set_should_not_wrap_parse_error_when_invalid_range(self): + """Test that a set-predicate bad range surfaces the raw parser error. + + Given: + An INTERSECTS ANY node with one unparseable range. + When: + Expanding it. + Then: + The raw RangeParser ValueError propagates *unwrapped* — the set- + predicate path does NOT apply the "Could not parse genomic range" + wrapper the single-operand path does. This pins the current + (pre-existing) asymmetry so any future unification is a conscious + change, not an accident. + """ + # Arrange + node, ctx = _context( + "SELECT * FROM a WHERE interval INTERSECTS ANY ('bad', 'chr1:1-2')", + {"this": _LEFT}, + ) + + # Act & assert + with pytest.raises(ValueError, match=r"Invalid genomic range format") as exc: + expand_spatial_set(node, ctx) + assert "Could not parse genomic range" not in str(exc.value) diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index a7141c4..9c8f8b0 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -11,7 +11,7 @@ from sqlglot import exp from sqlglot import parse_one -import giql # noqa: F401 (ensures the built-in expanders are registered) +import giql.expanders # noqa: F401 (registers built-in expanders) from giql import Table from giql import transpile from giql.canonicalizer import canonicalize_coordinates @@ -28,17 +28,16 @@ def _generate_through_passes(sql: str, tables: Tables) -> str: """Parse, run normalization passes 1-3, then generate SQL. Coordinate canonicalization for operator operands moved out of the emitter and - into the CanonicalizeCoordinates pass (issue #123), and DISTANCE generation - itself moved onto the registry's AST-expansion pass (epic #137, issue #140). - Emitter-level tests that pin canonicalized / expanded output must therefore - run all three passes before generating, exactly as - :func:`giql.transpile.transpile` does, rather than calling ``generate`` on a - bare parsed AST. The expansion pass only touches operators that opt in - (``GIQL_EXPAND``); operators still on the legacy emitter (NEAREST, the - spatial predicates) pass through untouched. This helper is used where the - full ``transpile`` pipeline would otherwise rewrite the node away (a - column-to-column ``INTERSECTS`` is turned into a binned equi-join before the - predicate emitter runs). + into the CanonicalizeCoordinates pass (issue #123), and DISTANCE (issue #140) + and the spatial / set predicates (issue #141) moved onto the registry's + ExpandOperators pass (epic #137). Emitter-level tests that pin canonicalized / + expanded output must therefore run all three passes before generating, exactly + as :func:`giql.transpile.transpile` does, rather than calling ``generate`` on a + bare parsed AST. Operators still on the legacy emitter (NEAREST, DISJOIN) pass + through the expansion pass untouched. This helper is used where the full + ``transpile`` pipeline would otherwise rewrite the node away (a column-to-column + ``INTERSECTS`` is turned into a binned equi-join before the predicate expander + runs). """ ast = parse_one(sql, dialect=GIQLDialect) ast = resolve_operator_refs(ast, tables) @@ -822,38 +821,33 @@ def test_giqldistance_canonicalizes_closed_ends_apart_from_gap_parity( ) assert output == expected - def test_error_handling_invalid_range(self): + def test_expand_intersects_should_raise_when_invalid_range(self): """ GIVEN invalid genomic range string in Intersects - WHEN intersects_sql is called + WHEN the INTERSECTS predicate is expanded THEN ValueError with descriptive message is raised. """ sql = "SELECT * FROM variants WHERE interval INTERSECTS 'invalid'" - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator() with pytest.raises(ValueError, match="Could not parse genomic range"): - generator.generate(ast) + _generate_through_passes(sql, Tables()) - def test_error_handling_unknown_operation(self): + def test_expand_intersects_should_raise_when_nonnumeric_range_bounds(self): """ - GIVEN unknown operation type in spatial operations - WHEN a spatial operation with unknown op_type is attempted - THEN ValueError is raised. + GIVEN an INTERSECTS range whose start/end bounds are non-numeric + WHEN the INTERSECTS predicate is expanded + THEN ValueError is raised from the range parse failure. - Note: This test verifies internal error handling by directly calling - a method with invalid input, which would only occur through code errors. + Note: 'chr:a-b' parses as a range shape but its bounds are not integers, + so the underlying RangeParser raises and the expander wraps it. (The + former "unknown operation" guard this exercised is now unreachable — + dispatch is closed over the three known op types — so this pins the + remaining reachable failure: a parse error on the literal range.) """ - # This is an indirect test - we verify the generator raises ValueError - # when given malformed range strings as that's how errors surface sql = "SELECT * FROM variants WHERE interval INTERSECTS 'chr:a-b'" - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator() with pytest.raises(ValueError): - generator.generate(ast) + _generate_through_passes(sql, Tables()) def test_select_sql_join_without_alias(self, tables_with_two_tables): """ diff --git a/tests/integration/datafusion/test_cross_target_oracle.py b/tests/integration/datafusion/test_cross_target_oracle.py index 6b717a1..3d4b864 100644 --- a/tests/integration/datafusion/test_cross_target_oracle.py +++ b/tests/integration/datafusion/test_cross_target_oracle.py @@ -3,9 +3,9 @@ These exercise the reusable oracle (``tests/integration/conftest.py``) over the operators that already emit identical generic SQL across Generic and DataFusion and run correctly on DuckDB: INTERSECTS (literal + column-to-column join), -CONTAINS, WITHIN, and standalone NEAREST. No operator has been migrated to the -expander registry yet (epic #137), so this lane locks in the verification path -every later migration (#140-#144) will consume. +CONTAINS, WITHIN, and standalone NEAREST. The spatial predicates have since been +migrated to the expander registry (#141, epic #137); this lane locks in the +verification path that migration and every later one (#142-#144) consume. For the non-join operators (DISTANCE, CONTAINS, WITHIN, ANY/ALL, CLUSTER, MERGE) the generic and datafusion targets emit byte-identical SQL and both run diff --git a/tests/test_expander.py b/tests/test_expander.py index 196571b..ab93732 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -1412,27 +1412,28 @@ def test_opted_in_restores_flag_after_exception(self): assert GIQLMerge.GIQL_EXPAND is False -class TestIEJoinEarlyReturnSkipsExpansion: - """Pin Finding 2: the duckdb IEJoin early return skips the ExpandOperators pass.""" - - @pytest.mark.xfail( - strict=True, - reason="#141: the duckdb IEJoin early return in transpile() emits before " - "ExpandOperators runs, so a flagged operator on an IEJoin-eligible query " - "is not expanded. Flips to pass when #141 runs expansion before the " - "early return (or defers the IEJoin transformer to the registry).", - ) - def test_iejoin_query_expands_flagged_operator(self, clean_registry): - """Test that an IEJoin-eligible duckdb query expands a flagged operator. +class TestIEJoinRegistryDeferral: + """The duckdb IEJoin path defers to a target-specific Intersects expander (#141). + + Resolves Finding 2: the IEJoin early return used to emit before the + ExpandOperators pass, so a flagged operator on an IEJoin-eligible query was + never expanded. Now a *target-specific* ``(DuckDBTarget, Intersects)`` + registry entry overrides the built-in join strategy entirely (the public + extension hook), while the default duckdb path — with no such override — + still emits the built-in IEJoin SQL. + """ + + def test_iejoin_query_expands_target_override_expander(self, clean_registry): + """Test that a target-specific Intersects override fires on an IEJoin query. Given: A column-to-column INTERSECTS join eligible for the duckdb IEJoin - path, with Intersects flagged GIQL_EXPAND and an expander registered. + path, with a (DuckDBTarget, Intersects) expander registered. When: Transpiling with dialect='duckdb'. Then: - The expander's sentinel should appear (currently it does NOT — the - IEJoin early return skips the pass; this xfail flips when #141 lands). + The override expander's sentinel should appear — the IEJoin path + defers to the registry rather than short-circuiting expansion. """ # Arrange clean_registry.register( @@ -1444,41 +1445,35 @@ def test_iejoin_query_expands_flagged_operator(self, clean_registry): ) # Act - with _opted_in(Intersects): - sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Assert assert "__giql_iejoin_sentinel" in sql + assert "SET VARIABLE __giql_iejoin_" not in sql - def test_iejoin_query_emits_legacy_sql_unchanged(self, clean_registry): - """Test that the legacy IEJoin SQL is emitted regardless of a flagged op. + def test_iejoin_query_emits_builtin_iejoin_without_override(self): + """Test that the default duckdb path emits the built-in IEJoin SQL. Given: - The same IEJoin-eligible duckdb query with Intersects flagged and an - expander registered. + The same IEJoin-eligible duckdb query and no target-specific + Intersects override registered (only the built-in generic expander). When: Transpiling with dialect='duckdb'. Then: - The legacy IEJoin SET VARIABLE SQL is emitted and the expander's - sentinel is absent (characterizing the current skip; the companion - xfail surfaces when #141 fixes it). + The built-in IEJoin SET VARIABLE SQL is emitted (the generic + predicate expander does not disable the join strategy). """ # Arrange - clean_registry.register( - DuckDBTarget(), Intersects, lambda n, c: exp.column("__giql_iejoin_sentinel") - ) query = ( "SELECT a.start FROM peaks a " "JOIN genes b ON a.interval INTERSECTS b.interval" ) # Act - with _opted_in(Intersects): - sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") + sql = transpile(query, tables=["peaks", "genes"], dialect="duckdb") # Assert assert "SET VARIABLE __giql_iejoin_" in sql - assert "__giql_iejoin_sentinel" not in sql class TestTranspileExpanderDispatch: From 6a4e33b5e2d7f5259254e7458acb83c464a9f6f3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 29 Jun 2026 14:20:09 -0400 Subject: [PATCH 095/142] fix: Migrate NEAREST to a registered expander with capability-driven fallback Move NEAREST off the legacy giqlnearest_sql emitter onto the operator-expander registry (epic #137 wave 3). Lateral-capable targets get the portable correlated LATERAL subquery; on DataFusion (no correlated-LATERAL physical plan) NEAREST expands to a decorrelated ROW_NUMBER() window fallback that returns identical rows, with a deterministic (start, end) tiebreaker and a synthesized subquery alias so an unaliased correlated NEAREST also runs there (and under python -O). Flip GIQL_EXPAND on GIQLNearest, delete giqlnearest_sql, SUPPORTS_LATERAL, and _nearest_resolution, and make the shared distance/nearest helpers staticmethods. Squashed rebase onto main (post-#156/#157) incorporating both review rounds: cross-target oracle coverage (k>1 ties, opposite-strand co-located rows, max_distance survivors), the SELECT * column-leak claim narrowed and xfail-pinned (tracked by #160), annotated helpers, reserved names derived from EXPAND_ALIAS_PREFIX, and the shared registry-seam reconciliation. --- docs/dialect/distance-operators.rst | 9 + src/giql/expanders/nearest.py | 474 ++++++++++++++++++ src/giql/expressions.py | 6 +- src/giql/generators/base.py | 201 +------- src/giql/targets.py | 12 +- tests/generators/test_base.py | 383 ++++++-------- .../datafusion/test_cross_target_oracle.py | 413 ++++++++++++++- tests/test_nearest_transpilation.py | 265 +++++++++- 8 files changed, 1313 insertions(+), 450 deletions(-) create mode 100644 src/giql/expanders/nearest.py diff --git a/docs/dialect/distance-operators.rst b/docs/dialect/distance-operators.rst index d3ebe89..ad333d4 100644 --- a/docs/dialect/distance-operators.rst +++ b/docs/dialect/distance-operators.rst @@ -309,6 +309,15 @@ Find nearby same-strand features within distance constraints: WHERE nearest.distance BETWEEN -10000 AND 10000 ORDER BY peaks.name, ABS(nearest.distance) +Target support +~~~~~~~~~~~~~~ + +A correlated ``NEAREST`` (its reference is an outer-row column) runs on lateral-capable engines — DuckDB and the generic target — via a correlated ``LATERAL`` subquery, and on Apache DataFusion, which has no correlated-``LATERAL`` physical plan, via a decorrelated window-function rewrite. For an **explicitly-projected** query (one that selects named columns, e.g. ``SELECT a.start, b.start, b.distance``) the two forms return identical results: the ``(start, end)`` tiebreaker orders rows tied at the k-th distance the same way on every engine, deterministically whenever ``(start, end)`` distinguishes the tied candidates. A standalone ``NEAREST`` with a literal reference is uncorrelated and uses the same ordered, limited subquery on every target. + +.. note:: + + **Known limitation —** ``SELECT *`` **/** ``SELECT b.*`` **over a correlated NEAREST on DataFusion.** The decorrelated window-function rewrite needs its reference-key and rank columns (``__giql_x_rk_*``, ``__giql_x_rn``) visible on the rewritten join, so a ``SELECT *`` or ``SELECT b.*`` over a correlated NEAREST exposes those reserved internal columns on DataFusion — a different output schema than the LATERAL form emits on DuckDB. The cross-target identity claim above therefore holds for **explicitly-projected** queries only. Projecting named columns avoids the leak entirely. A query-level wrapper that projects the reserved columns away on the DataFusion path is tracked by `#160 `_ (it depends on the query-level expander seam from #146). + Notes ~~~~~ diff --git a/src/giql/expanders/nearest.py b/src/giql/expanders/nearest.py new file mode 100644 index 0000000..482c232 --- /dev/null +++ b/src/giql/expanders/nearest.py @@ -0,0 +1,474 @@ +"""The NEAREST operator expander (epic #137, issue #142). + +NEAREST is the first operator whose expansion is genuinely capability-driven. +The portable form is a correlated ``LATERAL`` subquery: each outer row drives a +``SELECT ... FROM WHERE ORDER BY ABS(distance) +LIMIT k`` whose reference endpoints are outer-table columns. DuckDB and the +generic target plan that directly (``supports_lateral == True``). + +Apache DataFusion has no correlated-``LATERAL`` physical plan +(``supports_lateral == False``). For it the same k-nearest / ``max_distance`` / +``stranded`` / ``signed`` semantics are reproduced with a **decorrelated +window-function fallback**: the target is cross-joined against the outer +relation, each candidate is ranked with +``ROW_NUMBER() OVER (PARTITION BY ORDER BY ABS(distance))``, and +the surrounding ``CROSS JOIN LATERAL`` is rewritten into a plain join that +re-associates the top-``k`` ranked candidates back to every outer row sharing +that reference key. Ranking depends only on the reference value, so ranking once +per distinct reference value and re-joining is set-equivalent to the per-row +LATERAL form — deterministic when the ``(start, end)`` tiebreaker distinguishes +candidates tied at the k-th distance (verified by the cross-target result +oracle). + +A literal-reference (standalone) NEAREST is already an uncorrelated subquery, so +every target — DataFusion included — uses the LATERAL/standalone form unchanged; +only the *correlated* shape needs the fallback. + +The expander reuses :class:`giql.generators.base.BaseGIQLGenerator`'s +``_generate_distance_case`` (shared with DISTANCE, #140) and ``_nearest_*`` +passthrough/diagnostic helpers — all static, so they are called on the class with +no generator instance — then parses the assembled SQL fragments into AST so the +emitted SQL is reserialized by the active target's serializer. +""" + +from __future__ import annotations + +from sqlglot import exp +from sqlglot import parse_one + +from giql.dialect import GIQLDialect +from giql.expander import EXPAND_ALIAS_PREFIX +from giql.expander import ExpansionContext +from giql.expander import register +from giql.expressions import GIQLNearest +from giql.generators.base import BaseGIQLGenerator +from giql.resolver import ResolvedInterval +from giql.resolver import ResolvedRef +from giql.targets import GenericTarget + +#: Reserved column names the window-function fallback synthesizes inside its +#: ranked subquery. They are derived from the expander's reserved +#: ``EXPAND_ALIAS_PREFIX`` (``__giql_x_``) — rather than hardcoded — so they stay +#: clear of user identifiers and track the prefix if it ever changes, mirroring +#: the other reserved internal prefixes. +_RANK_COL = f"{EXPAND_ALIAS_PREFIX}rn" +_REF_KEY_PREFIX = f"{EXPAND_ALIAS_PREFIX}rk_" + + +def _nearest_params( + expression: GIQLNearest, +) -> tuple[int, int | None, bool, bool]: + """Unpack the (k, max_distance, stranded, signed) parameters of a NEAREST.""" + k = expression.args.get("k") + k_value = int(str(k)) if k else 1 + + max_distance = expression.args.get("max_distance") + max_dist_value = int(str(max_distance)) if max_distance else None + + is_stranded = BaseGIQLGenerator._extract_bool_param(expression.args.get("stranded")) + is_signed = BaseGIQLGenerator._extract_bool_param(expression.args.get("signed")) + return k_value, max_dist_value, is_stranded, is_signed + + +def _distance_and_filters( + expression: GIQLNearest, + table_name: str, + target_ref: ResolvedRef, + ref: ResolvedInterval, + ref_fragments: tuple[str, str, str, str | None] | None = None, +) -> tuple[str, str, list[str], str]: + """Build the shared distance SQL, the qualified target columns, and WHERE. + + Returns ``(distance_expr, abs_distance_expr, where_clauses, passthrough)`` — + the fragments common to the LATERAL/standalone form and the decorrelated + fallback. Distance math, the chromosome pre-filter, the optional strand match, + and the optional ``max_distance`` filter all reproduce the legacy + ``giqlnearest_sql`` emitter exactly. Each form derives its deterministic + ORDER BY tiebreaker from the target columns itself. + + ``ref_fragments`` optionally overrides the reference ``(chrom, start, end, + strand)`` SQL fragments. The LATERAL form consumes the resolution's + outer-qualified fragments verbatim; the fallback passes fragments pointing at + its renamed, pre-projected reference relation so the cross-joined columns + carry names distinct from the target's (DataFusion's planner cannot resolve a + window ordering over a join with duplicate column names). + """ + target_chrom, target_start, target_end = target_ref.cols + _k_value, max_dist_value, is_stranded, is_signed = _nearest_params(expression) + + output_table = BaseGIQLGenerator._nearest_output_encoding(expression, target_ref) + passthrough = BaseGIQLGenerator._nearest_passthrough( + table_name, target_start, target_end, output_table + ) + + if ref_fragments is not None: + ref_chrom, ref_start, ref_end, ref_strand_frag = ref_fragments + else: + ref_chrom, ref_start, ref_end, ref_strand_frag = ( + ref.chrom, + ref.start, + ref.end, + ref.strand, + ) + + ref_strand = None + target_strand = None + if is_stranded: + ref_strand = ref_strand_frag + if output_table and output_table.strand_col: + target_strand = f'{table_name}."{output_table.strand_col}"' + + target_chrom_expr = f'{table_name}."{target_chrom}"' + target_start_expr = f'{table_name}."{target_start}"' + target_end_expr = f'{table_name}."{target_end}"' + + distance_expr = BaseGIQLGenerator._generate_distance_case( + ref_chrom, + ref_start, + ref_end, + ref_strand, + target_chrom_expr, + target_start_expr, + target_end_expr, + target_strand, + stranded=is_stranded, + signed=is_signed, + ) + abs_distance_expr = f"ABS({distance_expr})" + + where_clauses = [f"{ref_chrom} = {target_chrom_expr}"] + if is_stranded and ref_strand and target_strand: + where_clauses.append(f"{ref_strand} = {target_strand}") + if max_dist_value is not None: + where_clauses.append(f"({abs_distance_expr}) <= {max_dist_value}") + + return distance_expr, abs_distance_expr, where_clauses, passthrough + + +def _lateral_form( + expression: GIQLNearest, + ctx: ExpansionContext, + table_name: str, + target_ref: ResolvedRef, + ref: ResolvedInterval, +) -> exp.Expression: + """The portable LATERAL/standalone subquery. + + Builds a two-level subquery: an inner ``SELECT , AS + distance FROM WHERE ...`` that materializes the distance, wrapped by + an outer ``SELECT * FROM () AS x ORDER BY ABS(x.distance), x., + x. LIMIT k`` that orders on the *precomputed* ``distance`` column. For a + correlated placement the parent ``LATERAL`` correlates it to the outer row; + for a standalone (literal-reference) placement it stands alone. + + Splitting the distance computation (inner) from the ordering (outer) is + load-bearing for cross-engine support: + + * DuckDB's correlated-``LATERAL`` binder will not resolve a SELECT-list alias + named ``distance`` from inside an ``ORDER BY`` that also projects + ``.*``, so the order key must reference a *materialized* column + (``x.distance``) from the wrapping level rather than an alias in the same + SELECT. + * DataFusion's planner, given the distance ``CASE`` re-emitted inline in the + ``ORDER BY`` over the chromosome-equality prefiltered scan, rewrites the + filtered ``chrom`` to a self-comparison in one copy of the CASE but not the + other and trips ``SanityCheckPlan``; ordering on the materialized column + avoids re-deriving the key. + + A ``(start, end)`` tiebreaker follows ``ABS(distance)`` so rows tied at the + k-th distance order deterministically — and identically across engines and + against the decorrelated fallback's ranking — *when ``(start, end)`` + distinguishes the tied candidates*. Two target rows sharing both distance and + ``(start, end)`` remain order-ambiguous (no key here breaks that residual + tie); the two forms are set-equivalent up to such coordinate-duplicate ties + (#142 A5). + """ + k_value, *_ = _nearest_params(expression) + ( + distance_expr, + _abs_distance_expr, + where_clauses, + passthrough, + ) = _distance_and_filters(expression, table_name, target_ref, ref) + where_sql = " AND ".join(where_clauses) + # The wrapping level reads the inner row's *bare* column names (the passthrough + # projected ``.*``), so the tiebreaker qualifies them by the wrapper + # alias, not the original ``table_name."col"``. + _chrom, target_start_col, target_end_col = target_ref.cols + wrapper = ctx.alias() + inner = ( + f"SELECT {passthrough}, {distance_expr} AS distance " + f"FROM {table_name} WHERE {where_sql}" + ) + sql = ( + f"(SELECT * FROM ({inner}) AS {wrapper} " + f'ORDER BY ABS({wrapper}."distance"), ' + f'{wrapper}."{target_start_col}", {wrapper}."{target_end_col}" ' + f"LIMIT {k_value})" + ) + return parse_one(sql, dialect=GIQLDialect) + + +def _outer_relation(ref: ResolvedInterval) -> tuple[str, str]: + """Return ``(physical_relation, alias)`` for the correlated reference table. + + The reference endpoints are alias-qualified fragments (``a."chrom"``). The + alias is the outer table's correlation name in the query; the physical + relation comes from the reference's backing :class:`~giql.table.Table`. Both + are needed to re-introduce the outer relation inside the decorrelated + subquery the fallback builds. + + The ``else alias`` branch (``ref.table is None``) only fires for a reference + whose backing table the resolver could not attach. For a *correlated* + NEAREST — the only shape that reaches the fallback — the reference is an + outer-row column, so the resolver always attaches its table and this branch + is not reached in practice. Falling back to ``alias`` (yielding a + cosmetically redundant ``FROM AS ``, where ``relation == + alias``) keeps the emitted SQL valid rather than emitting an empty relation + name should that invariant ever not hold; the assert pins the expectation. + """ + parsed = parse_one(ref.chrom, dialect=GIQLDialect) + alias = parsed.table if isinstance(parsed, exp.Column) else "" + if ref.table is not None: + relation = ref.table.name + else: + # Defensive fallback only: a correlated reference always carries a + # resolved backing table, so relation == alias here would be a redundant + # self-alias rather than a real two-name relation. + assert alias, ( + "correlated NEAREST fallback expected the reference to carry either a " + "backing table or an alias-qualified column" + ) + relation = alias + return relation, alias + + +def _fallback_form( + expression: GIQLNearest, + ctx: ExpansionContext, + table_name: str, + target_ref: ResolvedRef, + ref: ResolvedInterval, +) -> exp.Expression: + """The decorrelated window-function fallback for non-LATERAL targets. + + Rewrites the surrounding `` AS a CROSS JOIN LATERAL (nearest) AS b`` + into `` AS a JOIN () AS b ON AND + b. <= k``. The ranked subquery cross-joins the target against the outer + relation and ranks candidates per distinct reference key with + ``ROW_NUMBER()``; the join re-associates the top-k back to every outer row + sharing that key, reproducing the per-row LATERAL semantics. Swaps the parent + ``LATERAL`` for the decorrelated subquery in place and returns the (now + detached) NEAREST node, so the pass's own ``node.replace`` is a no-op. + + The no-op return relies on NEAREST having no nestable inner GIQL operator: a + detached node carrying a still-pending descendant would strand that + descendant's later ``node.replace``. NEAREST's only operands are a registered + target table and an interval reference, neither of which is an expandable + operator, so nothing pending hangs off the node this detaches. + """ + lateral = expression.parent + # Internal invariants the surrounding-AST rewrite depends on. The fallback + # only runs for a correlated NEAREST, whose pass-1 placement is always a CROSS + # JOIN LATERAL under a JOIN; a violation is an internal pipeline bug, not user + # error, so fail loudly with a clear message rather than dereferencing None. + # (The LATERAL *alias* is NOT an internal invariant — it is optional user + # input — so it is synthesized below rather than asserted; see B3.) + assert isinstance(lateral, exp.Lateral), ( + "correlated NEAREST fallback expected its parent to be a LATERAL, got " + f"{type(lateral).__name__}" + ) + join = lateral.parent + assert isinstance(join, exp.Join), ( + "correlated NEAREST fallback expected the LATERAL to sit under a JOIN, got " + f"{type(join).__name__}" + ) + # The decorrelated join references the LATERAL's alias on both sides (its ON + # clause and the replacement subquery's name). A correlated NEAREST written + # *without* a LATERAL alias is legitimate user input that transpiles fine on + # lateral-capable engines, so the fallback must not require one: synthesize a + # collision-safe alias from the run's sequence when the LATERAL carries none. + # This closes the DuckDB/DataFusion behavior gap (and, unlike an ``assert``, + # survives ``python -O``, which would otherwise strip the guard and leave a + # ``NoneType`` deref). The synthesized name is internal to the rewritten join. + lateral_alias = lateral.args.get("alias") + if lateral_alias is None or not lateral_alias.name: + alias = ctx.alias() + alias_node = exp.TableAlias(this=exp.to_identifier(alias)) + else: + alias = lateral_alias.name + alias_node = lateral_alias.copy() + + relation, outer_alias = _outer_relation(ref) + k_value, _max, is_stranded, _signed = _nearest_params(expression) + # Bare target column names: the candidate subquery exposes the target row via + # ``target.*``, so its tiebreaker columns are referenced by name, not by the + # ``table_name."col"`` qualifier the distance math uses. + _target_chrom, target_start_col, target_end_col = target_ref.cols + + # Pre-project the outer relation's reference columns under fresh, non-target + # names into a renamed derived relation. Cross-joining *this* (rather than the + # raw outer table) keeps every reference column distinct from the target's + # columns: DataFusion's planner cannot resolve a window ordering over a join + # whose two sides share column names (e.g. both expose ``start`` / ``end``). + # + # The reference key identifies one distinct reference interval, which the + # ranking partitions by and the join re-associates on. Position + # (chrom/start/end) alone identifies it in the unstranded case; in stranded + # mode strand joins the key too, because two outer rows at the same position + # but opposite strands must each get their own strand-filtered nearest. The + # ref relation is de-duplicated on the key with DISTINCT so ranking happens + # once per distinct reference and the join fans the top-k back out to every + # outer row sharing it — exactly the per-row LATERAL semantics, even when the + # outer table holds duplicate reference rows. + # Mint the synthetic relation aliases from the run's collision-safe sequence + # (rather than hardcoding ``__giql_x_ref`` / ``__giql_x_cand``) so two NEAREST + # operators in one query never reuse the same derived-relation name. The + # reserved *column* names below stay derived from EXPAND_ALIAS_PREFIX. + ref_relation_alias = ctx.alias() + candidate = ctx.alias() + strand_name = f"{_REF_KEY_PREFIX}strand" + stranded_key = is_stranded and ref.strand is not None + + key_names = [f"{_REF_KEY_PREFIX}chrom", f"{_REF_KEY_PREFIX}start", + f"{_REF_KEY_PREFIX}end"] + source_frags = [ref.chrom, ref.start, ref.end] + if stranded_key: + key_names.append(strand_name) + source_frags.append(ref.strand) + + ref_projection = ", ".join( + f'{frag} AS "{name}"' for name, frag in zip(key_names, source_frags) + ) + ref_relation = ( + f"(SELECT DISTINCT {ref_projection} FROM {relation} AS {outer_alias})" + f" AS {ref_relation_alias}" + ) + + # Reference fragments now point at the renamed relation's safe columns. + renamed = [f'{ref_relation_alias}."{name}"' for name in key_names] + renamed_strand = ( + f'{ref_relation_alias}."{strand_name}"' if stranded_key else None + ) + ref_fragments = (renamed[0], renamed[1], renamed[2], renamed_strand) + + ( + distance_expr, + _abs_distance_expr, + where_clauses, + passthrough, + ) = _distance_and_filters( + expression, table_name, target_ref, ref, ref_fragments=ref_fragments + ) + + # Surface the reference-key columns so the rewritten join can match each + # ranked candidate back to its outer row(s). Ranking depends only on these + # values, so partitioning by them and re-joining is identical to the per-row + # LATERAL form even when outer rows share a reference value. + key_cols = list(zip(key_names, renamed)) + key_projection = ", ".join(f'{frag} AS "{name}"' for name, frag in key_cols) + where_sql = " AND ".join(where_clauses) + + # Compute the candidate set (cross join + distance + reference keys) in an + # inner subquery, then add ROW_NUMBER() in the enclosing one. Keeping the + # join and the window in *separate* query levels is load-bearing on + # DataFusion: fused into one level its optimizer mis-derives the window's sort + # order from the chromosome-equality prefilter and trips ``SanityCheckPlan``. + inner = ( + f"SELECT {passthrough}, {distance_expr} AS distance, {key_projection} " + f"FROM {table_name} CROSS JOIN {ref_relation} " + f"WHERE {where_sql}" + ) + partition = ", ".join(f'{candidate}."{name}"' for name, _ in key_cols) + # A ``(start, end)`` tiebreaker follows ``ABS(distance)`` so rows tied at the + # k-th distance rank identically here and in the LATERAL form when + # ``(start, end)`` distinguishes them, making the two set-equivalent up to ties + # among candidates sharing both distance and coordinates (no engine-dependent + # tie ordering otherwise). + ranked = ( + f"(SELECT {candidate}.*, " + f"ROW_NUMBER() OVER (PARTITION BY {partition} " + f"ORDER BY ABS({candidate}.distance), " + f'{candidate}."{target_start_col}", {candidate}."{target_end_col}") ' + f'AS "{_RANK_COL}" ' + f"FROM ({inner}) AS {candidate})" + ) + ranked_subquery = parse_one(ranked, dialect=GIQLDialect) + + # Match each ranked candidate back to the *outer* relation by its reference + # value (the original outer-qualified fragments, e.g. ``a."chrom"``), not the + # renamed inner columns which exist only inside the subquery. + on_parts = [ + f'{alias}."{name}" = {src}' for name, src in zip(key_names, source_frags) + ] + on_parts.append(f'{alias}."{_RANK_COL}" <= {k_value}') + on_sql = " AND ".join(on_parts) + on_expr = parse_one(on_sql, dialect=GIQLDialect) + + subquery = exp.Subquery(this=ranked_subquery.this, alias=alias_node) + + # Convert `` CROSS JOIN LATERAL (nearest) AS b`` into + # `` JOIN (ranked) AS b ON AND b.rn <= k``. Swap the + # whole LATERAL out for the decorrelated subquery and drop the CROSS kind so + # the ON clause attaches as a plain (inner) join. + lateral.replace(subquery) + join.set("kind", None) + join.set("side", None) + join.set("on", on_expr) + + # The LATERAL (and the NEAREST node within it) is now detached; returning the + # node unchanged makes the pass's ``node.replace`` a no-op. + return expression + + +def expand_nearest(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: + """Expand a NEAREST node to LATERAL or the decorrelated window-function form. + + Selects on ``ctx.capabilities.supports_lateral`` and whether the node is + correlated (its parent is a ``LATERAL``). Lateral-capable targets and every + standalone (literal-reference) placement get the portable LATERAL/standalone + subquery; a correlated NEAREST on a target without LATERAL support gets the + decorrelated window-function fallback. + """ + assert isinstance(node, GIQLNearest) + resolution = ctx.resolution + + target_ref = resolution.slot("this") if resolution is not None else None + if not isinstance(target_ref, ResolvedRef): + # An unresolved target means it is not a registered table; raise the + # historical diagnostic (verbatim from the removed giqlnearest_sql). + target = node.this + if isinstance(target, exp.Table): + target_name = target.name + elif isinstance(target, exp.Column): + target_name = target.table if target.table else str(target.this) + else: + target_name = str(target) + raise ValueError( + f"Target table '{target_name}' not found in tables. " + "Register the table before transpiling." + ) + table_name = target_ref.name + + ref = resolution.slot("reference") + if not isinstance(ref, ResolvedInterval): + mode = BaseGIQLGenerator._detect_nearest_mode(node) + BaseGIQLGenerator._raise_nearest_reference_error(node, mode, resolution) + + # A literal-range reference is uncorrelated even under CROSS JOIN LATERAL: its + # endpoints are constants, not outer-row columns, so the subquery stands alone + # and every target — DataFusion included — takes the LATERAL/standalone form. + # Only a genuinely correlated reference (a column / implicit-outer endpoint) + # needs the decorrelated window-function fallback on a lateral-incapable + # target. Gating on parentage alone would mis-route a literal range into + # ``_fallback_form``, which dereferences a non-existent outer relation. + correlated = isinstance(node.parent, exp.Lateral) and ref.kind != "literal_range" + if correlated and not ctx.capabilities.supports_lateral: + return _fallback_form(node, ctx, table_name, target_ref, ref) + return _lateral_form(node, ctx, table_name, target_ref, ref) + + +# The generic registration covers every target through the registry's fallback +# chain; the expander branches on ctx.capabilities.supports_lateral internally, +# so no per-target override is needed. +register(GenericTarget, GIQLNearest)(expand_nearest) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 4949fdd..2e8fd0c 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -384,7 +384,11 @@ class GIQLNearest(exp.Func): #: half-open) operands are left untouched and the emitted SQL stays #: byte-identical. GIQL_CANONICALIZE = _CANONICALIZE - GIQL_EXPAND = _EXPAND + #: Migrated to the ExpandOperators pass (epic #137, issue #142): NEAREST is + #: expanded by ``giql.expanders.nearest`` — the portable correlated LATERAL + #: subquery where ``supports_lateral`` holds, a decorrelated window-function + #: form otherwise. The legacy ``giqlnearest_sql`` emitter has been removed. + GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index cc494dc..599eb0d 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -3,13 +3,13 @@ from giql.canonical import decanonical_end from giql.canonical import decanonical_start +from giql.dialect import GIQLDialect from giql.expressions import GIQLDisjoin from giql.expressions import GIQLNearest from giql.range_parser import RangeParser from giql.resolver import META_KEY from giql.resolver import OperatorResolution from giql.resolver import ResolvedColumn -from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef from giql.table import Table from giql.table import Tables @@ -28,10 +28,6 @@ class BaseGIQLGenerator(Generator): compatibility with virtually all SQL databases. """ - # Most databases support LATERAL joins (PostgreSQL 9.3+, DuckDB 0.7.0+) - # SQLite does not support LATERAL, so it overrides this to False - SUPPORTS_LATERAL = True - def __init__(self, tables: Tables | None = None, **kwargs): super().__init__(**kwargs) self.tables = tables or Tables() @@ -43,172 +39,9 @@ def __init__(self, tables: Tables | None = None, **kwargs): # ``spatialsetpredicate_sql`` emitters or their ``_generate_spatial_*`` / # ``_predicate_operand`` helpers. - def giqlnearest_sql(self, expression: GIQLNearest) -> str: - """Generate SQL for NEAREST function. - - Detects mode (standalone vs correlated) and generates appropriate SQL: - - Standalone: Direct query with ORDER BY + LIMIT - - Correlated (LATERAL): Subquery for k-nearest neighbors - - :param expression: - GIQLNearest expression node - :return: - SQL string for NEAREST operation - """ - # Detect mode - mode = self._detect_nearest_mode(expression) - - # Unpack the resolution metadata attached by ResolveOperatorRefs (pass 1). - resolution = self._nearest_resolution(expression) - - # Target (already a registered-table ResolvedRef from the pass). An - # unresolved target means it is not a registered table; raise the - # historical diagnostic. - target_ref = resolution.slot("this") if resolution is not None else None - if not isinstance(target_ref, ResolvedRef): - target = expression.this - if isinstance(target, exp.Table): - target_name = target.name - elif isinstance(target, exp.Column): - target_name = target.table if target.table else str(target.this) - else: - target_name = str(target) - raise ValueError( - f"Target table '{target_name}' not found in tables. " - "Register the table before transpiling." - ) - table_name = target_ref.name - target_chrom, target_start, target_end = target_ref.cols - - # The target's *declared* encoding, which the passed-through target row - # (SELECT {table_name}.*) must round-trip back into. CanonicalizeCoordinates - # (pass 2) preserves it on the resolution when it wraps a non-canonical - # target in a __giql_canon_* CTE (the slot's own Table is then None); a - # canonical target is left unwrapped and its slot Table carries the - # (identity) encoding. The synthesized `distance` column is encoding- - # invariant (a count of bases) and needs no round-trip. - output_table = self._nearest_output_encoding(expression, target_ref) - passthrough = self._nearest_passthrough( - table_name, target_start, target_end, output_table - ) - - # Reference interval (a ResolvedInterval from the pass). An unresolved - # reference re-raises the generator's historical diagnostic. Input - # canonicalization is owned by CanonicalizeCoordinates (pass 2, issue - # #123): a literal range is already canonical, and a column / implicit- - # outer reference's endpoints are canonicalized in place by the pass, so - # the emitter consumes the fragments verbatim with no canonicalization. - ref = resolution.slot("reference") - if not isinstance(ref, ResolvedInterval): - self._raise_nearest_reference_error(expression, mode, resolution) - ref_chrom, ref_start, ref_end = ref.chrom, ref.start, ref.end - - # Extract parameters - k = expression.args.get("k") - k_value = int(str(k)) if k else 1 # Default k=1 - - max_distance = expression.args.get("max_distance") - max_dist_value = int(str(max_distance)) if max_distance else None - - is_stranded = self._extract_bool_param(expression.args.get("stranded")) - is_signed = self._extract_bool_param(expression.args.get("signed")) - - # Resolve strand columns if stranded mode. The reference strand is - # carried on the resolved interval (a literal's strand, an explicit - # column's strand, or the outer table's strand for an implicit - # reference — already gated to preserve the historical divergence). - ref_strand = None - target_strand = None - if is_stranded: - ref_strand = ref.strand - # When pass 2 wraps a non-canonical target its slot Table is blanked, - # so the strand column name comes from the *declared* encoding the - # pass preserved (output_table). The canon CTE's SELECT * REPLACE - # passes the strand column through unchanged under its physical name, - # so the qualifier stays the relation NEAREST selects from. - if output_table and output_table.strand_col: - target_strand = f'{table_name}."{output_table.strand_col}"' - - # Distance math below assumes 0-based half-open. Input canonicalization - # is owned by CanonicalizeCoordinates (pass 2, issue #123): a - # non-canonical target is rewritten to a canonical __giql_canon_* CTE - # before generation (table_name then names the CTE), so the target - # endpoints are consumed verbatim with no in-emitter canonicalization. The - # output round-trip of the passed-through target row stays here (see the - # SELECT projection below). - target_start_expr = f'{table_name}."{target_start}"' - target_end_expr = f'{table_name}."{target_end}"' - - # Build distance calculation using CASE expression - # For NEAREST: ORDER BY absolute distance, but RETURN signed distance - distance_expr = self._generate_distance_case( - ref_chrom, - ref_start, - ref_end, - ref_strand, - f'{table_name}."{target_chrom}"', - target_start_expr, - target_end_expr, - target_strand, - stranded=is_stranded, - signed=is_signed, - ) - - # Use absolute distance for ordering and filtering - abs_distance_expr = f"ABS({distance_expr})" - - # Build WHERE clauses - where_clauses = [ - f'{ref_chrom} = {table_name}."{target_chrom}"' # Chromosome pre-filter - ] - - # Add strand matching for stranded mode - if is_stranded and ref_strand and target_strand: - where_clauses.append(f"{ref_strand} = {target_strand}") - - if max_dist_value is not None: - where_clauses.append(f"({abs_distance_expr}) <= {max_dist_value}") - - where_sql = " AND ".join(where_clauses) - - # Generate SQL based on mode - if mode == "standalone": - # Standalone mode: direct ORDER BY + LIMIT - # Return signed distance, but order by absolute distance - sql = f"""( - SELECT {passthrough}, {distance_expr} AS distance - FROM {table_name} - WHERE {where_sql} - ORDER BY {abs_distance_expr} - LIMIT {k_value} - )""" - else: - # Correlated mode: requires LATERAL join support - if not self.SUPPORTS_LATERAL: - raise ValueError( - "NEAREST in correlated mode (CROSS JOIN LATERAL) is not supported " - "in SQLite. SQLite does not support LATERAL joins. " - "\n\nAlternatives:" - "\n1. Use standalone mode: SELECT * FROM NEAREST(table, " - "reference='chr1:100-200', k=3)" - "\n2. Use DuckDB for queries requiring LATERAL joins" - "\n3. Manually write equivalent window function query" - ) - - # LATERAL mode: subquery for k-nearest neighbors - # Return signed distance, but order by absolute distance - sql = f"""( - SELECT {passthrough}, {distance_expr} AS distance - FROM {table_name} - WHERE {where_sql} - ORDER BY {abs_distance_expr} - LIMIT {k_value} - )""" - - return sql.strip() - + @staticmethod def _nearest_output_encoding( - self, expression: GIQLNearest, target_ref: ResolvedRef + expression: GIQLNearest, target_ref: ResolvedRef ) -> Table | None: """Return the target's declared encoding for NEAREST's row passthrough. @@ -233,8 +66,8 @@ def _nearest_output_encoding( return preserved return target_ref.table + @staticmethod def _nearest_passthrough( - self, table_name: str, target_start: str, target_end: str, @@ -454,8 +287,8 @@ def _disjoin_passthrough( f't.* REPLACE ({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' ) + @staticmethod def _generate_distance_case( - self, chrom_a: str, start_a: str, end_a: str, @@ -563,8 +396,9 @@ def _generate_distance_case( f"ELSE ({start_a} - {end_b} + 1) END END" ) + @staticmethod def _detect_nearest_mode( - self, expression: GIQLNearest, parent_expression: exp.Expression | None = None + expression: GIQLNearest, parent_expression: exp.Expression | None = None ) -> str: """Detect whether NEAREST is in standalone or correlated mode. @@ -588,25 +422,8 @@ def _detect_nearest_mode( # (validation will catch missing reference errors later) return "correlated" - def _nearest_resolution(self, expression: GIQLNearest) -> OperatorResolution | None: - """Return the NEAREST resolution attached by ResolveOperatorRefs (pass 1). - - The transpile pipeline attaches an - :class:`~giql.resolver.OperatorResolution` before generation, and it - survives the generator's defensive tree copy. The emitter reads only the - attached metadata; resolution lives entirely in the pass. - - :param expression: - GIQLNearest expression node - :return: - The attached :class:`~giql.resolver.OperatorResolution`, or ``None`` - if resolution did not produce one. - """ - resolution = expression.meta.get(META_KEY) - return resolution if isinstance(resolution, OperatorResolution) else None - + @staticmethod def _raise_nearest_reference_error( - self, expression: GIQLNearest, mode: str, resolution: OperatorResolution | None, @@ -654,7 +471,7 @@ def _raise_nearest_reference_error( # An explicit reference that deferred is a literal range that failed to # parse (column references always resolve). Re-parse to surface the # original parse error in the historical message. - reference_sql = self.sql(reference) + reference_sql = reference.sql(dialect=GIQLDialect) range_str = reference_sql.strip("'\"") try: RangeParser.parse(range_str).to_zero_based_half_open() diff --git a/src/giql/targets.py b/src/giql/targets.py index 6fd29d3..825a6d2 100644 --- a/src/giql/targets.py +++ b/src/giql/targets.py @@ -29,11 +29,13 @@ class Capabilities: Parameters ---------- supports_lateral : bool - Whether the engine supports ``LATERAL`` / correlated joins. Will - drive the NEAREST LATERAL-vs-window-function strategy (#142). Until - then, :attr:`giql.generators.base.BaseGIQLGenerator.SUPPORTS_LATERAL` - remains the live source of truth at generation time; #142 reconciles - the two. + Whether the engine supports ``LATERAL`` / correlated joins. Drives the + NEAREST LATERAL-vs-window-function strategy (#142): a correlated NEAREST + expands to a portable correlated ``LATERAL`` subquery where this holds + and to a decorrelated window-function form where it does not. This + capability is the single source of truth — the former + ``BaseGIQLGenerator.SUPPORTS_LATERAL`` generator attribute has been + removed. supports_star_replace : bool Whether the engine supports ``SELECT * REPLACE (...)``. Drives the coordinate-canonicalization output: ``* REPLACE`` where supported, diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index 9c8f8b0..3ac63c0 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -8,7 +8,6 @@ from hypothesis import given from hypothesis import settings from hypothesis import strategies as st -from sqlglot import exp from sqlglot import parse_one import giql.expanders # noqa: F401 (registers built-in expanders) @@ -17,7 +16,6 @@ from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expander import ExpandOperators -from giql.expressions import GIQLNearest from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables @@ -28,14 +26,14 @@ def _generate_through_passes(sql: str, tables: Tables) -> str: """Parse, run normalization passes 1-3, then generate SQL. Coordinate canonicalization for operator operands moved out of the emitter and - into the CanonicalizeCoordinates pass (issue #123), and DISTANCE (issue #140) - and the spatial / set predicates (issue #141) moved onto the registry's + into the CanonicalizeCoordinates pass (issue #123), and DISTANCE (#140), the + spatial / set predicates (#141), and NEAREST (#142) moved onto the registry's ExpandOperators pass (epic #137). Emitter-level tests that pin canonicalized / expanded output must therefore run all three passes before generating, exactly as :func:`giql.transpile.transpile` does, rather than calling ``generate`` on a - bare parsed AST. Operators still on the legacy emitter (NEAREST, DISJOIN) pass - through the expansion pass untouched. This helper is used where the full - ``transpile`` pipeline would otherwise rewrite the node away (a column-to-column + bare parsed AST. Operators still on the legacy emitter (DISJOIN) pass through + the expansion pass untouched. This helper is used where the full ``transpile`` + pipeline would otherwise rewrite the node away (a column-to-column ``INTERSECTS`` is turned into a binned equi-join before the predicate expander runs). """ @@ -110,13 +108,12 @@ def test_instantiation_defaults(self): """ GIVEN no tables provided WHEN Generator is instantiated with defaults - THEN Generator has empty Tables and SUPPORTS_LATERAL is True. + THEN Generator has empty Tables. """ generator = BaseGIQLGenerator() assert generator.tables is not None assert "variants" not in generator.tables - assert generator.SUPPORTS_LATERAL is True def test_instantiation_with_tables(self, tables_info): """ @@ -402,41 +399,43 @@ def test_spatialsetpredicate_sql_all(self): ) assert output == expected - def test_giqlnearest_sql_standalone(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_emit_ordered_limit_subquery_when_standalone( + self, tables_with_peaks_and_genes + ): """ GIVEN a GIQLNearest in standalone mode with literal reference - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs THEN Subquery with ORDER BY distance LIMIT k is generated. """ sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)" output = _generate_through_passes(sql, tables_with_peaks_and_genes) + # NEAREST now expands via its registered expander (#142): a two-level + # subquery — an inner SELECT that materializes ``distance`` and an outer + # wrapper that orders on the materialized ``__giql_x_0."distance"`` plus a + # deterministic ``(start, end)`` tiebreaker (#142 A5). Splitting the + # distance computation from the ordering keeps DuckDB's correlated-LATERAL + # binder and DataFusion's planner both happy while staying result- + # equivalent to the legacy single-level emitter. expected = ( - "SELECT * FROM (\n" - " SELECT genes.*, " - "CASE WHEN 'chr1' != genes.\"chrom\" THEN NULL " + "SELECT * FROM (SELECT * FROM (SELECT genes.*, " + "CASE WHEN 'chr1' <> genes.\"chrom\" THEN NULL " 'WHEN 1000 < genes."end" AND 2000 > genes."start" THEN 0 ' - 'WHEN 2000 <= genes."start" ' - 'THEN (genes."start" - 2000 + 1) ' - 'ELSE (1000 - genes."end" + 1) END AS distance\n' - " FROM genes\n" - " WHERE 'chr1' = genes.\"chrom\"\n" - " ORDER BY ABS(" - "CASE WHEN 'chr1' != genes.\"chrom\" THEN NULL " - 'WHEN 1000 < genes."end" AND 2000 > genes."start" THEN 0 ' - 'WHEN 2000 <= genes."start" ' - 'THEN (genes."start" - 2000 + 1) ' - 'ELSE (1000 - genes."end" + 1) END)\n' - " LIMIT 3\n" - " )" + 'WHEN 2000 <= genes."start" THEN (genes."start" - 2000 + 1) ' + 'ELSE (1000 - genes."end" + 1) END AS distance ' + "FROM genes WHERE 'chr1' = genes.\"chrom\") AS __giql_x_0 " + 'ORDER BY ABS(__giql_x_0."distance"), ' + '__giql_x_0."start", __giql_x_0."end" LIMIT 3)' ) assert output == expected - def test_giqlnearest_sql_correlated(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_emit_lateral_subquery_when_correlated( + self, tables_with_peaks_and_genes + ): """ GIVEN a GIQLNearest in correlated mode (LATERAL join context) - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs on a lateral-capable target THEN LATERAL-compatible subquery is generated. """ sql = ( @@ -446,33 +445,30 @@ def test_giqlnearest_sql_correlated(self, tables_with_peaks_and_genes): output = _generate_through_passes(sql, tables_with_peaks_and_genes) + # Reserialized by the #142 expander as a two-level subquery: the inner + # SELECT materializes ``distance`` (CASE and WHERE semantically unchanged) + # and the outer wrapper orders on the materialized + # ``__giql_x_0."distance"`` plus a deterministic ``(start, end)`` + # tiebreaker (#142 A5). expected = ( - "SELECT * FROM peaks CROSS JOIN LATERAL (\n" - " SELECT genes.*, " - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end" + 1) ' - 'ELSE (peaks."start" - genes."end" + 1) END AS distance\n' - " FROM genes\n" - ' WHERE peaks."chrom" = genes."chrom"\n' - " ORDER BY ABS(" - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' + "SELECT * FROM peaks CROSS JOIN LATERAL (SELECT * FROM (SELECT genes.*, " + 'CASE WHEN peaks."chrom" <> genes."chrom" THEN NULL ' + 'WHEN peaks."start" < genes."end" AND peaks."end" > genes."start" ' + 'THEN 0 WHEN peaks."end" <= genes."start" ' 'THEN (genes."start" - peaks."end" + 1) ' - 'ELSE (peaks."start" - genes."end" + 1) END)\n' - " LIMIT 3\n" - " )" + 'ELSE (peaks."start" - genes."end" + 1) END AS distance ' + 'FROM genes WHERE peaks."chrom" = genes."chrom") AS __giql_x_0 ' + 'ORDER BY ABS(__giql_x_0."distance"), ' + '__giql_x_0."start", __giql_x_0."end" LIMIT 3)' ) assert output == expected - def test_giqlnearest_sql_with_max_distance(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_filter_on_max_distance( + self, tables_with_peaks_and_genes + ): """ GIVEN a GIQLNearest with max_distance parameter - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs THEN WHERE clause includes distance filter. """ sql = ( @@ -483,40 +479,35 @@ def test_giqlnearest_sql_with_max_distance(self, tables_with_peaks_and_genes): output = _generate_through_passes(sql, tables_with_peaks_and_genes) + # Reserialized by the #142 expander as a two-level subquery; the + # max_distance filter on ABS of the distance CASE stays in the inner + # SELECT's WHERE and the outer wrapper orders on the materialized + # ``__giql_x_0."distance"`` plus the deterministic ``(start, end)`` + # tiebreaker (#142 A5). expected = ( - "SELECT * FROM peaks CROSS JOIN LATERAL (\n" - " SELECT genes.*, " - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' + "SELECT * FROM peaks CROSS JOIN LATERAL (SELECT * FROM (SELECT genes.*, " + 'CASE WHEN peaks."chrom" <> genes."chrom" THEN NULL ' + 'WHEN peaks."start" < genes."end" AND peaks."end" > genes."start" ' + 'THEN 0 WHEN peaks."end" <= genes."start" ' 'THEN (genes."start" - peaks."end" + 1) ' - 'ELSE (peaks."start" - genes."end" + 1) END AS distance\n' - " FROM genes\n" - ' WHERE peaks."chrom" = genes."chrom" ' - "AND (ABS(" - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' + 'ELSE (peaks."start" - genes."end" + 1) END AS distance ' + 'FROM genes WHERE peaks."chrom" = genes."chrom" ' + 'AND (ABS(CASE WHEN peaks."chrom" <> genes."chrom" THEN NULL ' + 'WHEN peaks."start" < genes."end" AND peaks."end" > genes."start" ' + 'THEN 0 WHEN peaks."end" <= genes."start" ' 'THEN (genes."start" - peaks."end" + 1) ' - 'ELSE (peaks."start" - genes."end" + 1) END)) <= 100000\n' - " ORDER BY ABS(" - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end" + 1) ' - 'ELSE (peaks."start" - genes."end" + 1) END)\n' - " LIMIT 5\n" - " )" + 'ELSE (peaks."start" - genes."end" + 1) END)) <= 100000) ' + 'AS __giql_x_0 ORDER BY ABS(__giql_x_0."distance"), ' + '__giql_x_0."start", __giql_x_0."end" LIMIT 5)' ) assert output == expected - def test_giqlnearest_sql_stranded(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_match_strand_when_stranded( + self, tables_with_peaks_and_genes + ): """ GIVEN a GIQLNearest with stranded := true - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs THEN Strand matching is included in WHERE clause. """ sql = ( @@ -527,45 +518,33 @@ def test_giqlnearest_sql_stranded(self, tables_with_peaks_and_genes): output = _generate_through_passes(sql, tables_with_peaks_and_genes) + # Reserialized by the #142 expander as a two-level subquery; the stranded + # distance CASE and the ``peaks.strand = genes.strand`` match in the inner + # WHERE are semantically unchanged, with the outer wrapper ordering on the + # materialized ``__giql_x_0."distance"`` plus the ``(start, end)`` + # tiebreaker (#142 A5). expected = ( - "SELECT * FROM peaks CROSS JOIN LATERAL (\n" - " SELECT genes.*, " - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' + "SELECT * FROM peaks CROSS JOIN LATERAL (SELECT * FROM (SELECT genes.*, " + 'CASE WHEN peaks."chrom" <> genes."chrom" THEN NULL ' 'WHEN peaks."strand" IS NULL OR genes."strand" IS NULL THEN NULL ' "WHEN peaks.\"strand\" = '.' OR peaks.\"strand\" = '?' THEN NULL " "WHEN genes.\"strand\" = '.' OR genes.\"strand\" = '?' THEN NULL " - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' + 'WHEN peaks."start" < genes."end" AND peaks."end" > genes."start" ' + 'THEN 0 WHEN peaks."end" <= genes."start" ' "THEN CASE WHEN peaks.\"strand\" = '-' " 'THEN -(genes."start" - peaks."end" + 1) ' 'ELSE (genes."start" - peaks."end" + 1) END ' "ELSE CASE WHEN peaks.\"strand\" = '-' " 'THEN -(peaks."start" - genes."end" + 1) ' - 'ELSE (peaks."start" - genes."end" + 1) END END AS distance\n' - " FROM genes\n" - ' WHERE peaks."chrom" = genes."chrom" ' - 'AND peaks."strand" = genes."strand"\n' - " ORDER BY ABS(" - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."strand" IS NULL OR genes."strand" IS NULL THEN NULL ' - "WHEN peaks.\"strand\" = '.' OR peaks.\"strand\" = '?' THEN NULL " - "WHEN genes.\"strand\" = '.' OR genes.\"strand\" = '?' THEN NULL " - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' - "THEN CASE WHEN peaks.\"strand\" = '-' " - 'THEN -(genes."start" - peaks."end" + 1) ' - 'ELSE (genes."start" - peaks."end" + 1) END ' - "ELSE CASE WHEN peaks.\"strand\" = '-' " - 'THEN -(peaks."start" - genes."end" + 1) ' - 'ELSE (peaks."start" - genes."end" + 1) END END)\n' - " LIMIT 3\n" - " )" + 'ELSE (peaks."start" - genes."end" + 1) END END AS distance ' + 'FROM genes WHERE peaks."chrom" = genes."chrom" ' + 'AND peaks."strand" = genes."strand") AS __giql_x_0 ' + 'ORDER BY ABS(__giql_x_0."distance"), ' + '__giql_x_0."start", __giql_x_0."end" LIMIT 3)' ) assert output == expected - def test_giqlnearest_sql_implicit_outer_without_strand_column(self): + def test_expand_nearest_should_skip_strand_when_outer_has_no_strand_column(self): """ GIVEN a stranded NEAREST whose implicit-outer table declares no strand column @@ -589,10 +568,12 @@ def test_giqlnearest_sql_implicit_outer_without_strand_column(self): assert "strand" not in output assert 'nostr."chrom" = genes."chrom"' in output - def test_giqlnearest_sql_signed(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_emit_signed_distance_when_signed( + self, tables_with_peaks_and_genes + ): """ GIVEN a GIQLNearest with signed := true - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs THEN Distance expression includes signed calculation. """ sql = ( @@ -603,57 +584,37 @@ def test_giqlnearest_sql_signed(self, tables_with_peaks_and_genes): output = _generate_through_passes(sql, tables_with_peaks_and_genes) + # Reserialized by the #142 expander as a two-level subquery; the signed + # distance CASE (negated ELSE branch for upstream) is semantically + # unchanged in the inner SELECT, with the outer wrapper ordering on the + # materialized ``__giql_x_0."distance"`` plus the ``(start, end)`` + # tiebreaker (#142 A5). expected = ( - "SELECT * FROM peaks CROSS JOIN LATERAL (\n" - " SELECT genes.*, " - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' - 'THEN (genes."start" - peaks."end" + 1) ' - 'ELSE -(peaks."start" - genes."end" + 1) END AS distance\n' - " FROM genes\n" - ' WHERE peaks."chrom" = genes."chrom"\n' - " ORDER BY ABS(" - 'CASE WHEN peaks."chrom" != genes."chrom" THEN NULL ' - 'WHEN peaks."start" < genes."end" ' - 'AND peaks."end" > genes."start" THEN 0 ' - 'WHEN peaks."end" <= genes."start" ' + "SELECT * FROM peaks CROSS JOIN LATERAL (SELECT * FROM (SELECT genes.*, " + 'CASE WHEN peaks."chrom" <> genes."chrom" THEN NULL ' + 'WHEN peaks."start" < genes."end" AND peaks."end" > genes."start" ' + 'THEN 0 WHEN peaks."end" <= genes."start" ' 'THEN (genes."start" - peaks."end" + 1) ' - 'ELSE -(peaks."start" - genes."end" + 1) END)\n' - " LIMIT 3\n" - " )" + 'ELSE -(peaks."start" - genes."end" + 1) END AS distance ' + 'FROM genes WHERE peaks."chrom" = genes."chrom") AS __giql_x_0 ' + 'ORDER BY ABS(__giql_x_0."distance"), ' + '__giql_x_0."start", __giql_x_0."end" LIMIT 3)' ) assert output == expected - def test_giqlnearest_sql_no_lateral_support(self, tables_with_peaks_and_genes): - """ - GIVEN a GIQLNearest on a generator with SUPPORTS_LATERAL=False - WHEN giqlnearest_sql is called in correlated mode - THEN ValueError is raised with helpful message. - """ - - # Create a generator subclass without LATERAL support - class NoLateralGenerator(BaseGIQLGenerator): - SUPPORTS_LATERAL = False - - # Use query without explicit reference to trigger correlated mode - sql = "SELECT * FROM peaks CROSS JOIN LATERAL NEAREST(genes, k := 3)" - ast = parse_one(sql, dialect=GIQLDialect) - ast = resolve_operator_refs(ast, tables_with_peaks_and_genes) - ast = canonicalize_coordinates(ast) - - generator = NoLateralGenerator(tables=tables_with_peaks_and_genes) - - with pytest.raises(ValueError, match="LATERAL"): - generator.generate(ast) + # The legacy ``SUPPORTS_LATERAL=False`` generator-level error path was removed + # with ``giqlnearest_sql`` (#142): lateral support is now a target capability, + # and a target without it (DataFusion) gets the decorrelated window-function + # fallback rather than a hard error. That fallback's result-identity with the + # LATERAL form is verified by the cross-target oracle + # (tests/integration/datafusion/test_cross_target_oracle.py). @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) @given( k=st.integers(min_value=1, max_value=100), max_distance=st.integers(min_value=1, max_value=10_000_000), ) - def test_giqlnearest_sql_parameter_handling_property( + def test_expand_nearest_should_carry_k_and_max_distance_property( self, tables_with_peaks_and_genes, k, max_distance ): """ @@ -866,12 +827,12 @@ def test_select_sql_join_without_alias(self, tables_with_two_tables): ) assert output == expected - def test_giqlnearest_sql_stranded_literal_with_strand( + def test_expand_nearest_should_use_literal_strand_when_stranded( self, tables_with_peaks_and_genes ): """ GIVEN a GIQLNearest with stranded := true and literal reference containing strand - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs THEN Strand from literal range is parsed and used in filtering. """ sql = ( @@ -885,12 +846,12 @@ def test_giqlnearest_sql_stranded_literal_with_strand( assert "'+'" in output assert 'genes."strand"' in output - def test_giqlnearest_sql_stranded_implicit_reference( + def test_expand_nearest_should_resolve_outer_strand_when_implicit_reference( self, tables_with_peaks_and_genes ): """ GIVEN a GIQLNearest in correlated mode with implicit reference and stranded := true - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs THEN Strand column is resolved from outer table and used. """ sql = "SELECT * FROM peaks CROSS JOIN LATERAL NEAREST(genes, k := 3, stranded := true)" @@ -989,29 +950,22 @@ def test_giqldistance_sql_literal_second_arg_error(self, tables_with_two_tables) with pytest.raises(ValueError, match="Literal range as second argument"): expander.transform(ast) - def test_giqlnearest_sql_missing_outer_table_error( + def test_expand_nearest_should_raise_when_outer_table_unresolvable( self, tables_with_peaks_and_genes ): """ - GIVEN a GIQLNearest in correlated mode without reference where outer table - cannot be found - WHEN giqlnearest_sql is called - THEN ValueError is raised with helpful message about specifying reference. + GIVEN a GIQLNearest without a reference and no resolvable outer table + WHEN the NEAREST expander runs + THEN ValueError is raised with a helpful message about specifying reference. """ + # Arrange — no reference and no LATERAL outer relation to infer one from. + sql = "SELECT * FROM NEAREST(genes, k := 3)" - nearest = GIQLNearest( - this=exp.Table(this=exp.Identifier(this="genes")), - k=exp.Literal.number(3), - ) - resolve_operator_refs(nearest, tables_with_peaks_and_genes) - canonicalize_coordinates(nearest) - - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - + # Act & assert with pytest.raises(ValueError, match="Could not find outer table"): - generator.giqlnearest_sql(nearest) + _generate_through_passes(sql, tables_with_peaks_and_genes) - def test_giqlnearest_sql_outer_table_not_in_tables(self): + def test_expand_nearest_should_raise_when_outer_table_unregistered(self): """ GIVEN a NEAREST whose implicit-outer relation is found but not registered WHEN the query is generated @@ -1027,10 +981,12 @@ def test_giqlnearest_sql_outer_table_not_in_tables(self): with pytest.raises(ValueError, match="not found in tables"): _generate_through_passes(sql, tables) - def test_giqlnearest_sql_invalid_reference_range(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_raise_when_reference_range_unparseable( + self, tables_with_peaks_and_genes + ): """ GIVEN a GIQLNearest with invalid/unparseable reference range string - WHEN giqlnearest_sql is called + WHEN the NEAREST expander runs THEN ValueError is raised with parse error details. """ sql = "SELECT * FROM NEAREST(genes, reference := 'invalid_range', k := 3)" @@ -1038,36 +994,35 @@ def test_giqlnearest_sql_invalid_reference_range(self, tables_with_peaks_and_gen with pytest.raises(ValueError, match="Could not parse reference genomic range"): _generate_through_passes(sql, tables_with_peaks_and_genes) - def test_giqlnearest_sql_no_tables_error(self): + def test_expand_nearest_should_raise_when_no_tables_registered(self): """ - GIVEN a GIQLNearest without tables registered - WHEN giqlnearest_sql is called - THEN ValueError is raised because target table cannot be resolved. + GIVEN a GIQLNearest with no tables registered + WHEN the NEAREST expander runs + THEN ValueError is raised because the target table cannot be resolved. """ + # Arrange sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)" - ast = parse_one(sql, dialect=GIQLDialect) - - # Generator with empty tables - table won't be found - generator = BaseGIQLGenerator() + # Act & assert with pytest.raises(ValueError, match="not found in tables"): - generator.generate(ast) + _generate_through_passes(sql, Tables()) - def test_giqlnearest_sql_target_not_in_tables(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_raise_when_target_unregistered( + self, tables_with_peaks_and_genes + ): """ - GIVEN a GIQLNearest with target table not registered - WHEN giqlnearest_sql is called - THEN ValueError is raised listing available tables. + GIVEN a GIQLNearest whose target table is not registered + WHEN the NEAREST expander runs + THEN ValueError is raised listing the unresolved table. """ + # Arrange sql = ( "SELECT * FROM NEAREST(unknown_table, reference := 'chr1:1000-2000', k := 3)" ) - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) + # Act & assert with pytest.raises(ValueError, match="not found in tables"): - generator.generate(ast) + _generate_through_passes(sql, tables_with_peaks_and_genes) def test_intersects_sql_unqualified_column(self): """ @@ -1085,57 +1040,47 @@ def test_intersects_sql_unqualified_column(self): ) assert output == expected - def test_giqlnearest_sql_stranded_unqualified_reference( + def test_expand_nearest_should_resolve_strand_when_reference_unqualified( self, tables_with_peaks_and_genes ): """ - GIVEN a GIQLNearest with stranded := true and unqualified column reference - WHEN giqlnearest_sql is called + GIVEN a GIQLNearest with stranded := true and an unqualified column reference + WHEN the NEAREST expander runs THEN Strand column is resolved without table prefix. """ - - # Create NEAREST with stranded=True and an unqualified column reference - # The reference is an unqualified column (no table prefix) - nearest = GIQLNearest( - this=exp.Table(this=exp.Identifier(this="genes")), - reference=exp.Column(this=exp.Identifier(this="interval")), - k=exp.Literal.number(3), - stranded=exp.Boolean(this=True), + # Arrange — the reference is an unqualified column (no table prefix). + sql = ( + "SELECT * FROM peaks CROSS JOIN LATERAL " + "NEAREST(genes, reference := interval, k := 3, stranded := true)" ) - resolve_operator_refs(nearest, tables_with_peaks_and_genes) - canonicalize_coordinates(nearest) - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.giqlnearest_sql(nearest) + # Act + output = _generate_through_passes(sql, tables_with_peaks_and_genes) - # Should produce valid output with unqualified strand column + # Assert assert "LIMIT 3" in output - # The strand column should be unqualified (no table prefix) assert '"strand"' in output - def test_giqlnearest_sql_identifier_target(self, tables_with_peaks_and_genes): + def test_expand_nearest_should_emit_ordered_subquery_for_literal_reference( + self, tables_with_peaks_and_genes + ): """ - GIVEN a GIQLNearest where target is an Identifier (not Table or Column) - WHEN giqlnearest_sql is called - THEN Target is converted to string and lookup proceeds. + GIVEN a GIQLNearest with a standalone literal reference + WHEN the NEAREST expander runs + THEN it produces a standalone ordered, limited subquery over the target + table with no correlated LATERAL. """ + # Arrange + sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)" - # Use exp.Identifier directly - not Table or Column - # This triggers the else branch at line 830 where str(target) is called - nearest = GIQLNearest( - this=exp.Identifier(this="genes"), - reference=exp.Literal.string("chr1:1000-2000"), - k=exp.Literal.number(3), - ) - resolve_operator_refs(nearest, tables_with_peaks_and_genes) - canonicalize_coordinates(nearest) - - generator = BaseGIQLGenerator(tables=tables_with_peaks_and_genes) - output = generator.giqlnearest_sql(nearest) + # Act + output = _generate_through_passes(sql, tables_with_peaks_and_genes) - # Should succeed and produce valid SQL + # Assert assert "genes" in output + assert "ORDER BY" in output assert "LIMIT 3" in output + assert "LATERAL" not in output @given( bool_repr=st.sampled_from(["true", "TRUE", "True", "1", "yes", "YES"]), @@ -1808,7 +1753,7 @@ def test_giqlnearest_should_canonicalize_reference_column_when_reference_is_one_ A 0-based half-open target table (bed_a) and an explicit reference column from a 1-based closed table (vcf_b). When: - giqlnearest_sql is called. + the NEAREST expander runs. Then: It should wrap the reference-side start as (start - 1), leave its end raw, and leave the target side raw. @@ -1841,7 +1786,7 @@ def test_giqlnearest_should_canonicalize_outer_table_columns_when_reference_is_i target table (bed_a) joined via CROSS JOIN LATERAL with no ``reference`` argument on NEAREST. When: - giqlnearest_sql is called. + the NEAREST expander runs. Then: It should canonicalize the outer table's columns based on vcf_b's convention — wrapping start as (vcf_b."start" - 1) and diff --git a/tests/integration/datafusion/test_cross_target_oracle.py b/tests/integration/datafusion/test_cross_target_oracle.py index 3d4b864..f67ff9b 100644 --- a/tests/integration/datafusion/test_cross_target_oracle.py +++ b/tests/integration/datafusion/test_cross_target_oracle.py @@ -14,13 +14,20 @@ genuinely divergent SQL across targets (the DuckDB IEJoin vs. the binned equi-join). -NEAREST's expansion uses a correlated ``LATERAL`` subquery, which DataFusion has -no physical plan for today; its generic-vs-duckdb equivalence case runs both on -DuckDB, and the full three-target oracle is pinned by a -``pytest.raises(match="OuterReferenceColumn")`` test (#142) that fails loudly on -an unrelated error and trips "DID NOT RAISE" — forcing conversion to a real -identity test — when DataFusion gains correlated LATERAL. DISJOIN has an -analogous pending-#153 gap (duplicate ``end`` output names). +NEAREST's correlated expansion is capability-driven (#142): lateral-capable +targets (generic, duckdb) emit the portable ``LATERAL`` subquery, while +DataFusion — which has no correlated-LATERAL physical plan — gets a decorrelated +window-function fallback. For **explicitly-projected** queries (those selecting +named columns) the two forms return identical rows, so the full three-target +identity oracle now runs on every target for that projection shape (the former +``_unsupported_pending_142`` ``pytest.raises`` pin has been promoted to a real +identity test). The identity claim is narrowed to explicit projections because a +``SELECT *`` / ``SELECT b.*`` over a correlated NEAREST on DataFusion additionally +exposes the fallback's reserved ``__giql_x_*`` rank/key columns, a divergent +output schema from the LATERAL form's — a known limitation pinned by the +``xfail`` ``test_correlated_nearest_star_projection_diverges_on_datafusion`` case +below and tracked for a query-level fix by #160 (dependent on #146). DISJOIN has +an analogous pending-#153 gap (duplicate ``end`` output names). """ import pytest @@ -204,32 +211,384 @@ def test_standalone_nearest_k1_agrees_generic_vs_duckdb_on_duckdb( engines={"generic": "duckdb"}, ) - def test_nearest_on_datafusion_unsupported_pending_142(self, cross_target_oracle): - """Test the full NEAREST oracle raises DataFusion's missing-LATERAL error. + def test_correlated_nearest_k1_agrees_across_all_targets(self, cross_target_oracle): + """Test correlated NEAREST k=1 returns identical rows on every target. Given: - The single-row NEAREST query and a candidate gene on chr1. + A single-row peaks table and three candidate genes at varying + distances on chr1. When: - The oracle runs all three targets — the datafusion target executes - the correlated LATERAL on DataFusion, which has no physical plan. + A correlated ``CROSS JOIN LATERAL NEAREST(..., k := 1)`` query runs + for every target — the generic and duckdb targets emit the portable + LATERAL form (executed on DuckDB, the lateral-capable engine), and + the datafusion target emits the decorrelated window-function fallback + the #142 expander produces (executed on DataFusion). Then: - DataFusion should raise its ``OuterReferenceColumn`` "not - implemented" error. This pins the known #142 gap: the ``match`` - narrows to the LATERAL signature so an unrelated/reworded DataFusion - error fails loudly, and a closed gap (no exception) trips pytest's - "DID NOT RAISE", forcing this to be converted into a real - cross-target identity test when DataFusion gains correlated LATERAL. + Every target should return the single nearest gene and agree. + + Promoted from the ``_unsupported_pending_142`` expected-failure pin: + DataFusion now plans correlated NEAREST through the capability-driven + window-function fallback, so the full three-target oracle is a real + identity test rather than a ``pytest.raises`` placeholder. The generic + target is routed to DuckDB because its portable SQL is the LATERAL form, + which only the datafusion-specific fallback decorrelates for DataFusion. """ # Arrange / Act / Assert - with pytest.raises(Exception, match="OuterReferenceColumn"): - cross_target_oracle( - "SELECT a.chrom, a.start AS a_start, b.start AS b_start " - "FROM peaks a " - "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", - peaks=[("chr1", 200, 300)], - genes=[("chr1", 280, 290)], - expected=[("chr1", 200, 280)], - ) + cross_target_oracle( + "SELECT a.chrom, a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + peaks=[("chr1", 200, 300)], + genes=[ + ("chr1", 1000, 1100), + ("chr1", 50, 60), + ("chr1", 280, 290), + ], + expected=[("chr1", 200, 280)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_k2_returns_two_nearest_across_targets( + self, cross_target_oracle + ): + """Test correlated NEAREST k=2 picks the two nearest on every target. + + Given: + One peak and four candidate genes, more than k of them on the peak's + chromosome at distinct distances. + When: + A correlated ``NEAREST(..., k := 2)`` runs on every target — DuckDB + via the LATERAL form, DataFusion via the decorrelated window fallback. + Then: + Every target should return the two nearest genes and agree, pinning + the top-k fan-out of the fallback against the LATERAL form. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 2) b", + peaks=[("chr1", 200, 300)], + genes=[ + ("chr1", 1000, 1100), + ("chr1", 50, 60), + ("chr1", 280, 290), + ("chr1", 310, 320), + ], + expected=[(200, 280), (200, 310)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_duplicate_reference_rows_fan_out( + self, cross_target_oracle + ): + """Test correlated NEAREST fans the top-k out to duplicate reference rows. + + Given: + Two identical peak rows and two candidate genes. + When: + A correlated ``NEAREST(..., k := 1)`` runs on every target. + Then: + Every target should return the nearest gene once per duplicate peak + (two rows), pinning the fallback's DISTINCT-then-rejoin fan-out so a + duplicate outer row is not collapsed. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + peaks=[("chr1", 200, 300), ("chr1", 200, 300)], + genes=[("chr1", 280, 290), ("chr1", 50, 60)], + expected=[(200, 280), (200, 280)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_partitions_by_chromosome(self, cross_target_oracle): + """Test correlated NEAREST keys the nearest per outer chromosome. + + Given: + Peaks on chr1 and chr2 and candidate genes on both chromosomes. + When: + A correlated ``NEAREST(..., k := 1)`` runs on every target. + Then: + Each peak should pair with the nearest gene on its own chromosome and + all targets agree, pinning the fallback's PARTITION BY reference key + across distinct outer keys. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.chrom AS chrom, a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + peaks=[("chr1", 200, 300), ("chr2", 200, 300)], + genes=[("chr1", 280, 290), ("chr2", 500, 510), ("chr2", 205, 215)], + expected=[("chr1", 200, 280), ("chr2", 200, 205)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_max_distance_boundary(self, cross_target_oracle): + """Test correlated NEAREST drops candidates beyond max_distance everywhere. + + Given: + A peak and two genes, one just inside and one far beyond a + ``max_distance`` threshold. + When: + A correlated ``NEAREST(..., k := 5, max_distance := 100)`` runs on + every target. + Then: + Every target should return only the in-threshold gene, pinning the + ``max_distance`` filter through both the LATERAL and fallback forms. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(" + "genes, reference := a.interval, k := 5, max_distance := 100) b", + peaks=[("chr1", 200, 300)], + genes=[("chr1", 360, 400), ("chr1", 5000, 5100)], + expected=[(200, 360)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_stranded_matches_strand(self, cross_target_oracle): + """Test stranded correlated NEAREST matches strand on every target. + + Given: + A ``+`` peak and two genes — a slightly farther ``+`` gene and a + nearer ``-`` gene. + When: + A correlated ``NEAREST(..., k := 1, stranded := true)`` runs on every + target. + Then: + Every target should return the same-strand (``+``) gene even though + the opposite-strand gene is nearer, in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(" + "genes, reference := a.interval, k := 1, stranded := true) b", + tables=[Table("peaks"), Table("genes")], + columns=_STRANDED_COLUMNS, + peaks=[("chr1", 200, 300, "+")], + genes=[("chr1", 280, 290, "+"), ("chr1", 250, 260, "-")], + expected=[(200, 280)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_signed_distance_agrees(self, cross_target_oracle): + """Test signed correlated NEAREST reports signed distances everywhere. + + Given: + A peak with one upstream and one downstream candidate gene. + When: + A correlated ``NEAREST(..., k := 2, signed := true)`` projects the + ``distance`` column on every target. + Then: + Every target should report a negative distance for the upstream gene + and a positive one for the downstream gene, in agreement. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start, b.distance AS d " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(" + "genes, reference := a.interval, k := 2, signed := true) b", + peaks=[("chr1", 200, 300)], + genes=[("chr1", 50, 60), ("chr1", 360, 400)], + expected=[(200, 50, -141), (200, 360, 61)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_k_th_distance_tie_breaks_on_coordinates( + self, cross_target_oracle + ): + """Test the (start, end) tiebreaker picks the same k-th candidate everywhere. + + Given: + One peak and three genes where two candidates are tied at the k-th + (k=1) distance — both 100 bp away, one upstream and one downstream of + the peak — so only the ``(start, end)`` tiebreaker can choose between + them (the lower ``(start, end)`` wins). + When: + A correlated ``NEAREST(..., k := 1)`` runs on every target — DuckDB via + the LATERAL form's ``ORDER BY ABS(distance), start, end LIMIT 1`` and + DataFusion via the fallback's matching ``ROW_NUMBER()`` ordering. + Then: + Every target should return the lower-coordinate tied candidate (the + upstream gene), so the LATERAL and window forms agree on the tie rather + than ordering it by engine-dependent chance. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + peaks=[("chr1", 200, 300)], + # Upstream gene ends at 100 (gap 100); downstream gene starts at 400 + # (gap 100). Both tie at distance 100; (start, end) breaks the tie in + # favor of the upstream gene (start 50 < start 400). + genes=[("chr1", 50, 100), ("chr1", 400, 450)], + expected=[(200, 50)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_stranded_opposite_strands_same_position( + self, cross_target_oracle + ): + """Test stranded NEAREST keys per-strand for co-located opposite-strand rows. + + Given: + Two peaks at the *same* position but on opposite strands (``+`` and + ``-``), and one same-position ``+`` gene plus one ``-`` gene, so the + strand-augmented reference key must keep each outer row's nearest + strand-matched independently. + When: + A correlated ``NEAREST(..., k := 1, stranded := true)`` runs on every + target — DuckDB via the LATERAL form, DataFusion via the fallback whose + reference key includes strand. + Then: + The ``+`` peak should pair with the ``+`` gene and the ``-`` peak with + the ``-`` gene, in agreement, proving the fan-out keys by strand so two + co-located opposite-strand outer rows are not collapsed. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.strand AS a_strand, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(" + "genes, reference := a.interval, k := 1, stranded := true) b", + tables=[Table("peaks"), Table("genes")], + columns=_STRANDED_COLUMNS, + peaks=[("chr1", 200, 300, "+"), ("chr1", 200, 300, "-")], + genes=[("chr1", 280, 290, "+"), ("chr1", 250, 260, "-")], + expected=[("+", 280), ("-", 250)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_max_distance_keeps_k_survivors( + self, cross_target_oracle + ): + """Test max_distance with k>1 keeps every in-threshold survivor everywhere. + + Given: + One peak and four genes where three sit within a ``max_distance`` + threshold at distinct distances and one sits beyond it, with k larger + than the survivor count. + When: + A correlated ``NEAREST(..., k := 3, max_distance := 200)`` runs on + every target — DuckDB via the LATERAL form, DataFusion via the window + fallback. + Then: + Every target should return exactly the three in-threshold genes (the + beyond-threshold gene dropped), proving ``max_distance`` and the top-k + survivor set interact identically across both forms. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start, b.start AS b_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(" + "genes, reference := a.interval, k := 3, max_distance := 200) b", + peaks=[("chr1", 200, 300)], + genes=[ + ("chr1", 350, 360), + ("chr1", 420, 430), + ("chr1", 480, 490), + ("chr1", 5000, 5100), + ], + expected=[(200, 350), (200, 420), (200, 480)], + engines={"generic": "duckdb"}, + ) + + def test_correlated_nearest_unaliased_lateral_agrees_across_targets( + self, cross_target_oracle + ): + """Test an unaliased correlated NEAREST agrees across targets (B3 on-engine). + + Given: + A correlated ``CROSS JOIN LATERAL NEAREST(...)`` written *without* a + table alias — legitimate GIQL that, before B3, raised on DataFusion + while running on DuckDB. + When: + The query runs on every target — DuckDB via the LATERAL form and + DataFusion via the decorrelated fallback, which now synthesizes the + missing alias instead of asserting one. + Then: + Every target should return the single nearest gene and agree, proving + the synthesized-alias fallback both transpiles and executes on the real + DataFusion engine. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT a.start AS a_start " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1)", + peaks=[("chr1", 200, 300)], + genes=[ + ("chr1", 1000, 1100), + ("chr1", 50, 60), + ("chr1", 280, 290), + ], + expected=[(200,)], + engines={"generic": "duckdb"}, + ) + + @pytest.mark.xfail( + strict=True, + reason="#160: SELECT b.* over a correlated NEAREST on DataFusion exposes " + "the decorrelated fallback's reserved __giql_x_rk_*/__giql_x_rn columns, " + "so the output schema diverges from the LATERAL form's on DuckDB. The " + "cross-target identity claim is narrowed to explicitly-projected queries " + "until #160 (dependent on #146) adds a query-level wrapper that projects " + "the reserved columns away on the DataFusion path; this flips to a real " + "identity test then.", + ) + def test_correlated_nearest_star_projection_diverges_on_datafusion( + self, cross_target_oracle + ): + """Test SELECT b.* over a correlated NEAREST diverges per target (pins #160). + + Given: + A single-row peak and three candidate genes on chr1. + When: + A correlated ``NEAREST(..., k := 1)`` query projects ``b.*`` on every + target — DuckDB emits the LATERAL form (``genes.* + distance``) while + DataFusion's fallback additionally surfaces the reserved + ``__giql_x_rk_*`` / ``__giql_x_rn`` columns on ``b``. + Then: + The cross-target row sets should NOT agree (DataFusion's rows carry the + extra reserved columns), so the oracle's identity assertion fails. This + xfail pins the known divergence so it is not silently forgotten; it + flips to a passing identity test when #160 hides the reserved columns. + """ + # Arrange / Act & Assert (xfail: identity assertion is expected to fail) + cross_target_oracle( + "SELECT b.* " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + peaks=[("chr1", 200, 300)], + genes=[ + ("chr1", 1000, 1100), + ("chr1", 50, 60), + ("chr1", 280, 290), + ], + expected=[("chr1", 280, 290, 0)], + engines={"generic": "duckdb"}, + ) + + +#: A chrom/start/end/strand schema for the stranded NEAREST oracle cases (the +#: default oracle schema carries no strand column). +_STRANDED_COLUMNS = ( + ("chrom", "utf8"), + ("start", "int64"), + ("end", "int64"), + ("strand", "utf8"), +) class TestCrossTargetOracleIntersectsAnyAll: diff --git a/tests/test_nearest_transpilation.py b/tests/test_nearest_transpilation.py index 2488cb0..9caac74 100644 --- a/tests/test_nearest_transpilation.py +++ b/tests/test_nearest_transpilation.py @@ -7,26 +7,48 @@ import pytest from sqlglot import parse_one +import giql.expanders # noqa: F401 (side-effect: registers the NEAREST expander) from giql import Table from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect +from giql.expander import ExpandOperators from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables +from giql.targets import DataFusionTarget +from giql.targets import GenericTarget + + +def _generate_for_target(sql: str, tables: Tables, target) -> str: + """Parse, run passes 1-3 against *target*, then generate SQL. + + Drives the expander for a specific :class:`~giql.targets.Target` so a + capability-dependent shape (e.g. DataFusion's decorrelated window fallback, + chosen because ``supports_lateral`` is False) can be asserted without an + engine. + """ + ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables) + ast = canonicalize_coordinates(ast) + ast = ExpandOperators(target, tables).transform(ast) + return BaseGIQLGenerator(tables=tables).generate(ast) def _generate(sql: str, tables: Tables) -> str: - """Parse, run normalization passes 1 and 2, then generate SQL. + """Parse, run normalization passes 1-3, then generate SQL. - Operator resolution and coordinate canonicalization moved out of the emitter - and into the ResolveOperatorRefs / CanonicalizeCoordinates passes (epic #114, - issues #118-#123). Emitter-level tests must run both passes before generating, - exactly as :func:`giql.transpile.transpile` does, rather than calling - ``generate`` on a bare parsed AST. + Operator resolution, coordinate canonicalization, and operator expansion + moved out of the emitter into the ResolveOperatorRefs / CanonicalizeCoordinates + / ExpandOperators passes (epics #114, #137). NEAREST is now produced by its + registered expander (issue #142) rather than a ``giqlnearest_sql`` emitter, so + these tests must run pass 3 before generating, exactly as + :func:`giql.transpile.transpile` does, rather than calling ``generate`` on a + bare parsed AST. """ ast = parse_one(sql, dialect=GIQLDialect) ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) + ast = ExpandOperators(GenericTarget(), tables).transform(ast) return BaseGIQLGenerator(tables=tables).generate(ast) @@ -160,3 +182,234 @@ def test_nearest_with_signed(self, tables_with_peaks_and_genes): assert "ELSE -(" in output, ( f"Expected signed distance with negation for upstream, got:\n{output}" ) + + +class TestNearestDataFusionFallbackShape: + """Engine-free transpile-shape checks for the DataFusion window fallback (A8). + + A correlated NEAREST on the DataFusion target (``supports_lateral`` is False) + expands to the decorrelated window-function form. These assert its structural + invariants without running an engine: the window is present, the top-k filter + is a ``<= k`` predicate, no correlated ``LATERAL`` survives, and the candidate + cross-join and the window live at separate query levels. + """ + + def test_fallback_emits_window_with_topk_and_no_lateral( + self, tables_with_peaks_and_genes + ): + """Test the DataFusion fallback emits a windowed top-k with no LATERAL. + + Given: + A correlated NEAREST(genes, k := 1) on the DataFusion target. + When: + Transpiling. + Then: + It should emit a ROW_NUMBER() window, a `<= 1` top-k predicate, no + surviving LATERAL, and the cross-join and window at separate query + levels. + """ + # Arrange + sql = ( + "SELECT * FROM peaks " + "CROSS JOIN LATERAL NEAREST(genes, reference := peaks.interval, k := 1) AS b" + ) + + # Act + output = _generate_for_target( + sql, tables_with_peaks_and_genes, DataFusionTarget() + ) + + # Assert + assert "ROW_NUMBER(" in output.upper() + assert "OVER (" in output.upper() + assert "<= 1" in output + assert "LATERAL" not in output.upper() + # The candidate cross-join sits one level below the window: the window's + # FROM is a parenthesized subquery, so a CROSS JOIN appears nested inside. + assert "CROSS JOIN" in output.upper() + + def test_fallback_stranded_emits_window_and_strand_match( + self, tables_with_peaks_and_genes + ): + """Test the stranded DataFusion fallback keeps a strand match, no LATERAL. + + Given: + A stranded correlated NEAREST on the DataFusion target. + When: + Transpiling. + Then: + It should emit the window form, keep a strand equality in the + candidate WHERE, and surface no LATERAL. + """ + # Arrange + sql = ( + "SELECT * FROM peaks CROSS JOIN LATERAL " + "NEAREST(genes, reference := peaks.interval, k := 1, stranded := true) AS b" + ) + + # Act + output = _generate_for_target( + sql, tables_with_peaks_and_genes, DataFusionTarget() + ) + + # Assert + assert "ROW_NUMBER(" in output.upper() + assert "LATERAL" not in output.upper() + assert 'peaks."strand"' in output + assert 'genes."strand"' in output + + def test_fallback_k_greater_than_one_uses_k_in_topk_predicate( + self, tables_with_peaks_and_genes + ): + """Test the DataFusion fallback carries the requested k in its top-k filter. + + Given: + A correlated NEAREST(genes, k := 3) on the DataFusion target. + When: + Transpiling. + Then: + The top-k predicate should carry the requested k (`<= 3`) rather than a + LIMIT, and no LATERAL should survive. + """ + # Arrange + sql = ( + "SELECT * FROM peaks " + "CROSS JOIN LATERAL NEAREST(genes, reference := peaks.interval, k := 3) AS b" + ) + + # Act + output = _generate_for_target( + sql, tables_with_peaks_and_genes, DataFusionTarget() + ) + + # Assert + assert "ROW_NUMBER(" in output.upper() + assert "<= 3" in output + assert "LATERAL" not in output.upper() + + +class TestNearestUnaliasedCorrelatedFallback: + """The fallback synthesizes a LATERAL alias when the user omits one (B3).""" + + def test_expand_nearest_should_synthesize_alias_when_correlated_lateral_unaliased_on_nonlateral_target( # noqa: E501 + self, tables_with_peaks_and_genes + ): + """Test an unaliased correlated NEAREST transpiles on a non-LATERAL target. + + Given: + A correlated NEAREST whose surrounding CROSS JOIN LATERAL carries no + table alias — legitimate GIQL that transpiles fine on lateral-capable + engines — targeted at DataFusion (``supports_lateral`` is False), which + takes the decorrelated window fallback. + When: + Transpiling for the DataFusion target. + Then: + It should not raise: the fallback synthesizes an alias via + ``ctx.alias()`` instead of asserting one is present, and emits the + decorrelated window form (a synthesized ``__giql_x_`` alias, no + LATERAL). + """ + # Arrange + sql = ( + "SELECT a.start FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1)" + ) + + # Act + output = _generate_for_target( + sql, tables_with_peaks_and_genes, DataFusionTarget() + ) + + # Assert + assert "ROW_NUMBER(" in output.upper() + assert "LATERAL" not in output.upper() + assert "__giql_x_" in output + + def test_expand_nearest_should_not_assert_on_unaliased_lateral_under_O(self): + """Test the unaliased correlated fallback survives ``python -O``. + + Given: + A fresh ``python -O`` interpreter (asserts stripped), in which an + asserted alias precondition would degrade to a ``NoneType`` deref + rather than a clear error. + When: + Transpiling an unaliased correlated NEAREST for the DataFusion dialect. + Then: + It should transpile without raising and emit the window fallback, + proving the alias is synthesized (not asserted) so the optimized + interpreter cannot strip the guard into an opaque crash. + """ + # Arrange + import subprocess + import sys + + code = ( + "from giql import transpile, Table; " + "sql = transpile(" + "'SELECT a.start FROM peaks a CROSS JOIN LATERAL " + "NEAREST(genes, reference := a.interval, k := 1)', " + "tables=[Table('peaks'), Table('genes')], dialect='datafusion'); " + "assert 'ROW_NUMBER(' in sql.upper(), sql; " + "assert 'LATERAL' not in sql.upper(), sql; " + "print('ok')" + ) + + # Act + result = subprocess.run( + [sys.executable, "-O", "-c", code], + capture_output=True, + text=True, + ) + + # Assert + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "ok" + + +class TestNearestFallbackDetachContract: + """The fallback detaches its NEAREST node so the pass's replace is a no-op (A10).""" + + def test_fallback_detaches_node_and_rewritten_join_survives_pass( + self, tables_with_peaks_and_genes + ): + """Test the fallback detaches the NEAREST subtree and leaves the rewritten join. + + Given: + A correlated NEAREST on the DataFusion target, captured before the + ExpandOperators pass runs. + When: + Running the pass (which dispatches to the decorrelated fallback). + Then: + The original NEAREST node should be detached from the returned tree — + its surrounding LATERAL is swapped out, so it no longer reaches the + result root, making the pass's own ``node.replace`` a no-op — and the + rewritten plain JOIN (no LATERAL, ROW_NUMBER window present) should + survive in the returned tree. + """ + # Arrange + from giql.expressions import GIQLNearest + + sql = ( + "SELECT a.start FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) AS b" + ) + ast = parse_one(sql, dialect=GIQLDialect) + ast = resolve_operator_refs(ast, tables_with_peaks_and_genes) + ast = canonicalize_coordinates(ast) + nearest = ast.find(GIQLNearest) + assert nearest is not None and nearest.root() is ast + + # Act + result = ExpandOperators( + DataFusionTarget(), tables_with_peaks_and_genes + ).transform(ast) + + # Assert + # The fallback swaps out the LATERAL holding the NEAREST, so the node's + # subtree is detached: it no longer reaches the result root, which is what + # makes the pass's ``node.replace`` a no-op. + assert nearest.root() is not result + assert not list(result.find_all(GIQLNearest)) + output = BaseGIQLGenerator(tables=tables_with_peaks_and_genes).generate(result) + assert "LATERAL" not in output.upper() + assert "ROW_NUMBER(" in output.upper() From 1aecfa3dae4eb663f0c8b7e05844e2f99dd56eb7 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 29 Jun 2026 15:00:50 -0400 Subject: [PATCH 096/142] refactor: Migrate DISJOIN to a registered expander and fix #153 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move DISJOIN off the legacy giqldisjoin_sql emitter onto the operator-expander registry (epic #137 wave 3), the last operator migration. The expander assembles the __giql_dj_* WITH-CTE subquery as AST and selects the full-row passthrough by capability: SELECT * REPLACE where supports_star_replace holds (DuckDB), the portable * EXCEPT projection otherwise (DataFusion family). Flip GIQL_EXPAND on GIQLDisjoin and delete giqldisjoin_sql and its DISJOIN-only helpers from the generator. Also fix the duplicate-column bug: alias all four columns in every __giql_dj_cuts UNION branch so the de-canonicalized end column no longer collides with the end-cut under one output name, which DataFusion rejected — promoting the previously pending cross-target DISJOIN case to a real three-target identity test. Squashed rebase onto main (post-#156/#157/#158) incorporating both review rounds: non-canonical * EXCEPT oracle coverage, an engine-free cuts-CTE alias regression, the DJ_PREFIX constant shared with the resolver, parse_one over maybe_parse, typed expander node, refreshed comments, and the shared registry-seam reconciliation. --- src/giql/constants.py | 9 + src/giql/expanders/disjoin.py | 348 ++++++++++++++++++ src/giql/expressions.py | 6 +- src/giql/generators/base.py | 269 +------------- src/giql/resolver.py | 7 +- src/giql/targets.py | 9 + src/giql/transpile.py | 2 +- .../integration/coordinate_space/conftest.py | 5 +- .../datafusion/test_cross_target_oracle.py | 116 ++++-- tests/test_canonicalizer.py | 10 + tests/test_disjoin_transpilation.py | 95 ++++- tests/test_usage_patterns.py | 7 +- 12 files changed, 556 insertions(+), 327 deletions(-) create mode 100644 src/giql/expanders/disjoin.py diff --git a/src/giql/constants.py b/src/giql/constants.py index 0daf016..51a0943 100644 --- a/src/giql/constants.py +++ b/src/giql/constants.py @@ -12,3 +12,12 @@ # Default bin size for INTERSECTS binned equi-join optimization DEFAULT_BIN_SIZE = 10_000 + +#: The reserved alias prefix naming DISJOIN's internal CTEs (``__giql_dj_ref`` / +#: ``__giql_dj_tgt`` / ``__giql_dj_cuts`` / ...). Follows the same +#: reserved-prefix scheme as :data:`giql.canonicalizer.CANON_PREFIX`; the leading +#: double underscore keeps +#: the namespace clear of user identifiers. A single source of truth so the +#: resolver's reserved-prefix guard and the DISJOIN expander's CTE names and +#: guard cannot drift apart. +DJ_PREFIX = "__giql_dj_" diff --git a/src/giql/expanders/disjoin.py b/src/giql/expanders/disjoin.py new file mode 100644 index 0000000..f784805 --- /dev/null +++ b/src/giql/expanders/disjoin.py @@ -0,0 +1,348 @@ +"""The DISJOIN operator expander (epic #137, wave 3 — issue #143). + +DISJOIN splits each target interval at every reference breakpoint strictly +interior to it, passing the full target row through and appending the +sub-interval as ``disjoin_chrom`` / ``disjoin_start`` / ``disjoin_end`` in the +target table's declared coordinate system. A coverage filter drops sub-intervals +overlapping no reference interval; an omitted reference defaults to the target +set. + +This module is the AST-expansion replacement for the legacy +``BaseGIQLGenerator.giqldisjoin_sql`` emitter. It assembles the same WITH-CTE +subquery, parses it back into a sqlglot :class:`~sqlglot.expressions.Expression`, +and returns that node so the active target's serializer renders it — dissolving +the emit-time string special-case left by epic #114. + +Three behaviours are capability-driven / corrected here relative to the legacy +emitter: + +* **star-REPLACE portability (#143).** The full-row passthrough de-canonicalizes + the interval columns back into the target's declared encoding. On a target that + supports ``SELECT * REPLACE (...)`` (``supports_star_replace`` — DuckDB) the + passthrough emits ``t.* REPLACE (...)``. On a target without it (the generic + baseline and the DataFusion family) it emits the portable + ``t.* EXCEPT (...)`` form plus the two recomputed interval columns. This + portable form is **not** SQL-92 — it requires ``SELECT * EXCEPT (...)`` + support, which the DataFusion family provides but a strict SQL-92 engine (and + DuckDB, which prefers ``* REPLACE``) does not — so the generic non-canonical + passthrough is **not DuckDB-runnable** and runs only on an ``* EXCEPT``-capable + engine. +* **passthrough column order diverges across targets (issue #143).** The two + passthrough forms are *row-equivalent* but **not column-order-equivalent**. + ``* REPLACE`` substitutes the start/end columns *in place*, so the row keeps + its original column order. ``* EXCEPT (start, end)`` drops those two columns + and the recomputed ones are *re-appended* at the end of the projection, so on + the generic / DataFusion path the start/end columns move to the right of any + passthrough column. A ``SELECT *`` over a non-canonical DISJOIN is therefore + column-order-divergent between DuckDB and the generic target; only the row + *set* is identical. Select **explicit** columns (in a chosen order) when the + output column order must agree across targets. +* **duplicate-column fix (#153).** Every UNION branch of the ``__giql_dj_cuts`` + CTE aliases all four projected columns (``kc`` / ``ks`` / ``ke`` / ``pos``), so + the ``t."end"`` column and the end-cut expression never collide on an output + name when ``end`` de-canonicalizes to the bare physical column (the default + 0-based half-open identity case). DuckDB tolerated the prior unaliased + duplicate; DataFusion rejected it as a non-unique projection name. + +Input canonicalization is owned by ``CanonicalizeCoordinates`` (pass 2, #122): +every non-canonical interval-bearing operand is rewritten to a canonical +``__giql_canon_*`` CTE before this pass runs, so the expander consumes +already-canonical 0-based half-open columns and applies no input-canonicalization +arithmetic. The output round-trip back to the target's declared encoding stays +here, driven by the original encoding pass 2 preserves on the resolution. +""" + +from __future__ import annotations + +from sqlglot import exp +from sqlglot import parse_one + +from giql.canonical import decanonical_end +from giql.canonical import decanonical_start +from giql.constants import DJ_PREFIX +from giql.dialect import GIQLDialect +from giql.expander import ExpansionContext +from giql.expander import register +from giql.expressions import GIQLDisjoin +from giql.resolver import OperatorResolution +from giql.resolver import ResolvedRef +from giql.table import Table +from giql.targets import GenericTarget + + +@register(GenericTarget, GIQLDisjoin) +def expand_disjoin(node: GIQLDisjoin, ctx: ExpansionContext) -> exp.Expression: + """Expand a DISJOIN node to its WITH-CTE subquery AST for the active target. + + Registered for :class:`~giql.targets.GenericTarget`, so it is the portable + fallback every target resolves to through the registry's generic chain. The + one capability branch — ``ctx.capabilities.supports_star_replace`` — selects + the passthrough projection form, so a single expander covers both the + ``* REPLACE`` (DuckDB) and ``* EXCEPT`` (DataFusion / generic) targets. + + Parameters + ---------- + node : GIQLDisjoin + The :class:`~giql.expressions.GIQLDisjoin` node being expanded. + ctx : ExpansionContext + The expansion context carrying the node's pass-1/pass-2 resolution + metadata, the active target and its capabilities, and the registered + tables. + + Returns + ------- + exp.Expression + The parsed AST of the parenthesized WITH-CTE subquery that replaces the + DISJOIN node. + """ + sql = _build_disjoin_sql(node, ctx) + return parse_one(sql, dialect=GIQLDialect) + + +def _build_disjoin_sql(node: GIQLDisjoin, ctx: ExpansionContext) -> str: + """Assemble the DISJOIN WITH-CTE subquery SQL string for the active target. + + Mirrors the legacy ``BaseGIQLGenerator.giqldisjoin_sql`` shape one named + ``__giql_dj_*`` CTE at a time, with the #153 alias fix and the + capability-driven passthrough applied. + """ + resolution = ctx.resolution + target_ref, ref, ref_from = _disjoin_resolution(node, resolution) + target_name = target_ref.name + target_chrom, target_start, target_end = target_ref.cols + + ref_chrom, ref_start, ref_end = ref.cols + is_self_reference = ref.coverage_skippable + + # The target's *declared* encoding, which disjoin_* output and the + # passed-through interval round-trip back into. Pass 2 preserves it on the + # resolution when it wraps a non-canonical target (the slot's own Table is + # then None); a canonical target keeps the (identity) encoding on its slot. + output_table = _disjoin_output_encoding(resolution, target_ref) + + # Post-pass every operand is canonical 0-based half-open, so the physical + # columns are consumed verbatim with no input-canonicalization arithmetic. + t_chrom = f't."{target_chrom}"' + t_start = f't."{target_start}"' + t_end = f't."{target_end}"' + + # Reference endpoints: unqualified for the breakpoint CTE, qualified by 'r' + # for the coverage EXISTS filter. + bp_start = f'"{ref_start}"' + bp_end = f'"{ref_end}"' + r_start = f'r."{ref_start}"' + r_end = f'r."{ref_end}"' + + # disjoin_start / disjoin_end are emitted in the target's declared coordinate + # system so an output row carries one convention; the cut math stays canonical + # internally. + out_start = decanonical_start("s.seg_start", output_table) + out_end = decanonical_end("s.seg_end", output_table) + passthrough = _disjoin_passthrough(ctx, target_start, target_end, output_table) + + # Build the WITH clause one named fragment per __giql_dj_* CTE so each block + # reads on its own. The `seg_end > seg_start` guard in the final WHERE is + # belt-and-suspenders: UNION already dedupes cut positions, so LEAD cannot + # produce a zero-length segment unless it becomes UNION ALL. + ref_cte = f"__giql_dj_ref AS (SELECT * FROM {ref_from})" + tgt_cte = f"__giql_dj_tgt AS (SELECT * FROM {target_name})" + bp_cte = ( + "__giql_dj_bp AS (" + f'SELECT "{ref_chrom}" AS chrom, {bp_start} AS pos FROM __giql_dj_ref ' + "UNION " + f'SELECT "{ref_chrom}" AS chrom, {bp_end} AS pos FROM __giql_dj_ref)' + ) + # #153: alias all four columns in EVERY UNION branch. The output names already + # come from branch 1, so this is behaviour-preserving on DuckDB and makes each + # branch's projection internally unique for strict engines (DataFusion), where + # an unaliased t."end" would otherwise collide with the end-cut t."end". + cuts_cte = ( + "__giql_dj_cuts AS (" + f'SELECT t."{target_chrom}" AS kc, t."{target_start}" AS ks, ' + f't."{target_end}" AS ke, {t_start} AS pos FROM __giql_dj_tgt AS t ' + "UNION " + f'SELECT t."{target_chrom}" AS kc, t."{target_start}" AS ks, ' + f't."{target_end}" AS ke, {t_end} AS pos FROM __giql_dj_tgt AS t ' + "UNION " + f'SELECT t."{target_chrom}" AS kc, t."{target_start}" AS ks, ' + f't."{target_end}" AS ke, bp.pos AS pos ' + "FROM __giql_dj_tgt AS t JOIN __giql_dj_bp AS bp " + f"ON bp.chrom = {t_chrom} AND bp.pos > {t_start} " + f"AND bp.pos < {t_end})" + ) + segs_cte = ( + "__giql_dj_segs AS (" + "SELECT kc, ks, ke, pos AS seg_start, " + "LEAD(pos) OVER (PARTITION BY kc, ks, ke ORDER BY pos) AS seg_end " + "FROM __giql_dj_cuts)" + ) + # In self-reference mode the coverage EXISTS is provably always true: every + # emitted segment lies inside its parent target row, and that row is itself a + # member of the reference set. Skip the clause so the planner does not waste + # work on a no-op semi-join. The __giql_dj_ref CTE itself stays live because + # __giql_dj_bp still draws breakpoints from it. + where_clauses = ["s.seg_end IS NOT NULL", "s.seg_end > s.seg_start"] + if not is_self_reference: + where_clauses.append( + f'EXISTS (SELECT 1 FROM __giql_dj_ref AS r WHERE r."{ref_chrom}" = s.kc ' + f"AND {r_start} <= s.seg_start AND {r_end} > s.seg_start)" + ) + where_sql = " AND ".join(where_clauses) + final_select = ( + f"SELECT {passthrough}, s.kc AS disjoin_chrom, " + f"{out_start} AS disjoin_start, " + f"{out_end} AS disjoin_end FROM __giql_dj_tgt AS t " + f'JOIN __giql_dj_segs AS s ON t."{target_chrom}" = s.kc ' + f'AND t."{target_start}" = s.ks AND t."{target_end}" = s.ke ' + f"WHERE {where_sql}" + ) + return ( + f"(WITH {ref_cte}, {tgt_cte}, {bp_cte}, " + f"{cuts_cte}, {segs_cte} {final_select})" + ) + + +def _disjoin_passthrough( + ctx: ExpansionContext, + target_start: str, + target_end: str, + output_table: Table | None, +) -> str: + """Project the target's full row, de-canonicalizing the interval columns. + + When the target's declared encoding is canonical 0-based half-open the row + passes through as a plain ``t.*`` — the identity fast path, portable on every + engine. When it is non-canonical the interval columns (canonical inside + ``__giql_dj_tgt``) are de-canonicalized back into that encoding: + + * ``t.* REPLACE (...)`` on a target that supports it (``supports_star_replace`` + — DuckDB), substituting start/end **in place**; + * the portable ``t.* EXCEPT (start, end), , `` form otherwise + (the generic baseline / DataFusion family), which every ``* EXCEPT``-capable + engine plans. This form is **not** SQL-92 and is **not DuckDB-runnable**: it + requires ``SELECT * EXCEPT`` support. + + Column-order divergence (issue #143) + ------------------------------------ + The two forms produce row-equivalent but **not column-order-equivalent** + output. ``* REPLACE`` keeps the original column order; ``* EXCEPT`` drops + start/end and **re-appends** the recomputed columns at the end of the + projection, so they move to the right of any passthrough column on the + generic / DataFusion path. A ``SELECT *`` over a non-canonical DISJOIN is + therefore order-divergent across targets — only the row set agrees. Select + explicit columns when the cross-target column order must match. + """ + if output_table is None or ( + output_table.coordinate_system == "0based" + and output_table.interval_type == "half_open" + ): + return "t.*" + pt_start = decanonical_start(f't."{target_start}"', output_table) + pt_end = decanonical_end(f't."{target_end}"', output_table) + if ctx.capabilities.supports_star_replace: + return ( + f't.* REPLACE ({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' + ) + # Portable substitution: drop the two interval columns from the star and + # re-project them recomputed. EXCEPT removes them from the row, the trailing + # projections add them back in the target's encoding under their own names. + return ( + f't.* EXCEPT ("{target_start}", "{target_end}"), ' + f'{pt_start} AS "{target_start}", {pt_end} AS "{target_end}"' + ) + + +def _disjoin_output_encoding( + resolution: OperatorResolution | None, target_ref: ResolvedRef +) -> Table | None: + """Return the target's declared encoding for DISJOIN's output round-trip. + + ``CanonicalizeCoordinates`` (pass 2) records the original + :class:`~giql.table.Table` on the resolution when it wraps a non-canonical + target (blanking the slot's own ``table``). For an unwrapped target — a + canonical registered table, or any target when the pass did not run — the + slot's own ``table`` carries the (identity) encoding. + """ + if isinstance(resolution, OperatorResolution): + preserved = resolution.output_tables.get("this") + if preserved is not None: + return preserved + return target_ref.table + + +def _disjoin_resolution( + node: GIQLDisjoin, resolution: OperatorResolution | None +) -> tuple[ResolvedRef, ResolvedRef, str]: + """Unpack the DISJOIN resolution attached by ResolveOperatorRefs (pass 1). + + Returns ``(target_ref, ref, ref_from)`` where ``ref_from`` is the text + following ``FROM`` inside the ``__giql_dj_ref`` CTE. A subquery reference + carries no name, so it is rendered from the AST node as an aliased derived + table; registered tables and CTEs are selected from by name. + + The resolver pass deliberately leaves unresolvable slots unresolved; for + those, and for a resolved name using the reserved ``__giql_dj_`` prefix, this + re-raises DISJOIN's historical diagnostics verbatim. + """ + target_ref = ( + resolution.slot("this") + if isinstance(resolution, OperatorResolution) + else None + ) + + # An unresolved target means it is not a registered table. + if target_ref is None: + target = node.this + if isinstance(target, exp.Table): + target_name = target.name + elif isinstance(target, exp.Column): + target_name = target.table if target.table else str(target.this) + else: + target_name = str(target) + raise ValueError( + f"Target table '{target_name}' not found in tables. " + "Register the table before transpiling." + ) + + # The __giql_dj_ prefix names the operator's internal CTEs; a target table + # using it would collide with them. + if target_ref.name.startswith(DJ_PREFIX): + raise ValueError( + f"DISJOIN target {target_ref.name!r} uses the reserved " + "'__giql_dj_' prefix, which names the operator's internal " + "CTEs. Rename the table." + ) + + reference = node.args.get("reference") + ref = resolution.slot("reference") + if ref is not None: + if ref.kind == "subquery": + # Serializing the subquery with sqlglot's stock generator (not + # BaseGIQLGenerator) is safe because nested GIQL operators expand + # deepest-first: any GIQL construct inside this operand is already + # rewritten to standard AST by the time this outer DISJOIN expands, + # so the stock generator only ever round-trips plain SQL here. + ref_sql = reference.sql(dialect=GIQLDialect) + return target_ref, ref, f"{ref_sql} AS __giql_dj_rs" + return target_ref, ref, ref.name + + # Unresolved reference: re-classify it and raise the matching historical + # diagnostic. An omitted reference always resolves, so reference is non-None. + if not isinstance(reference, (exp.Table, exp.Column, exp.Identifier)): + raise ValueError( + "DISJOIN reference must be a table name, a CTE, or a " + f"(SELECT ...) subquery; got {type(reference).__name__}: " + f"{reference}" + ) + ref_name = reference.name + if ref_name.startswith(DJ_PREFIX): + raise ValueError( + f"DISJOIN reference {ref_name!r} uses the reserved " + "'__giql_dj_' prefix, which names the operator's internal " + "CTEs. Rename the reference relation." + ) + raise ValueError( + f"DISJOIN reference {ref_name!r} is neither a registered table " + "nor a CTE defined in this query. Register the table or define " + "the CTE before transpiling." + ) diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 2e8fd0c..6a0f440 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -438,7 +438,11 @@ class GIQLDisjoin(exp.Func): #: (0-based half-open) operands are left unwrapped and the emitted SQL stays #: byte-identical. GIQL_CANONICALIZE = True - GIQL_EXPAND = _EXPAND + #: Opt DISJOIN into the ExpandOperators pass (epic #137, issue #143). DISJOIN + #: is migrated to a registered expander (``giql.expanders.disjoin``), so the + #: pass replaces the node with the expander's AST and the legacy + #: ``giqldisjoin_sql`` emitter is removed. + GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index 599eb0d..b0f3f79 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -4,7 +4,6 @@ from giql.canonical import decanonical_end from giql.canonical import decanonical_start from giql.dialect import GIQLDialect -from giql.expressions import GIQLDisjoin from giql.expressions import GIQLNearest from giql.range_parser import RangeParser from giql.resolver import META_KEY @@ -106,187 +105,17 @@ def _nearest_passthrough( return f"{table_name}.*" pt_start = decanonical_start(f'{table_name}."{target_start}"', output_table) pt_end = decanonical_end(f'{table_name}."{target_end}"', output_table) + # TODO(#142): this emits an unconditional ``* REPLACE`` (DuckDB-only). + # When DataFusion gains correlated LATERAL, adopt the capability branch the + # DISJOIN expander uses (``giql.expanders.disjoin._disjoin_passthrough``): + # ``* REPLACE`` where ``supports_star_replace`` holds, the portable + # ``* EXCEPT`` form otherwise, so a non-canonical NEAREST passthrough runs + # on the DataFusion family too. return ( f"{table_name}.* REPLACE " f'({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' ) - def giqldisjoin_sql(self, expression: GIQLDisjoin) -> str: - """Generate SQL for the DISJOIN table function. - - DISJOIN splits each target interval at every reference breakpoint - strictly interior to it. The full target row passes through unchanged; - the sub-interval is appended as ``disjoin_chrom`` / ``disjoin_start`` / - ``disjoin_end`` in the target table's coordinate system. A coverage - filter drops sub-intervals overlapping no reference interval. When no - ``reference`` is given it defaults to the target set. - - Input canonicalization is owned by ``CanonicalizeCoordinates`` (pass 2, - issue #122): every non-canonical interval-bearing operand is rewritten to - a canonical ``__giql_canon_*`` CTE before generation, so this emitter - consumes already-canonical 0-based half-open columns and applies no - in-emitter canonicalization arithmetic. The output round-trip back to the - target's declared encoding stays here — the ``disjoin_*`` columns and the - passed-through interval are synthesized at generation time and cannot be - reached by a pass-2 outermost-projection rewrite — driven by the original - encoding the pass preserves on the resolution. - - :param expression: - GIQLDisjoin expression node - :return: - SQL string (a parenthesized WITH-CTE subquery) for the DISJOIN table - """ - # Unpack the resolution metadata attached by ResolveOperatorRefs (pass 1) - # and rewritten by CanonicalizeCoordinates (pass 2). - target_ref, ref, ref_from = self._disjoin_resolution(expression) - target_name = target_ref.name - target_chrom, target_start, target_end = target_ref.cols - ref_chrom, ref_start, ref_end = ref.cols - is_self_reference = ref.coverage_skippable - - # The target's *declared* encoding, which disjoin_* output and the - # passed-through interval must round-trip back into. Pass 2 preserves it - # on the resolution when it wraps a non-canonical target (the slot's own - # Table is then None); a canonical target is left unwrapped and its slot - # Table carries the (identity) encoding. - output_table = self._disjoin_output_encoding(expression, target_ref) - - # Post-pass every operand is canonical 0-based half-open (a registered - # table is either identity-encoded or rewritten to a canonical CTE), so - # the physical columns are consumed verbatim with no canonicalization. - t_chrom = f't."{target_chrom}"' - t_start = f't."{target_start}"' - t_end = f't."{target_end}"' - - # Reference endpoints: unqualified for the breakpoint CTE, qualified by - # 'r' for the coverage EXISTS filter. - bp_start = f'"{ref_start}"' - bp_end = f'"{ref_end}"' - r_start = f'r."{ref_start}"' - r_end = f'r."{ref_end}"' - - # disjoin_start / disjoin_end are emitted in the target's declared - # coordinate system so an output row carries one convention; the cut math - # stays canonical internally. - out_start = decanonical_start("s.seg_start", output_table) - out_end = decanonical_end("s.seg_end", output_table) - passthrough = self._disjoin_passthrough(target_start, target_end, output_table) - - # Build the WITH clause one named fragment per __giql_dj_* CTE so each - # block reads on its own. The `seg_end > seg_start` guard in the final - # WHERE is belt-and-suspenders: UNION already dedupes cut positions, so - # LEAD cannot produce a zero-length segment unless it becomes UNION ALL. - ref_cte = f"__giql_dj_ref AS (SELECT * FROM {ref_from})" - tgt_cte = f"__giql_dj_tgt AS (SELECT * FROM {target_name})" - bp_cte = ( - "__giql_dj_bp AS (" - f'SELECT "{ref_chrom}" AS chrom, {bp_start} AS pos FROM __giql_dj_ref ' - "UNION " - f'SELECT "{ref_chrom}" AS chrom, {bp_end} AS pos FROM __giql_dj_ref)' - ) - cuts_cte = ( - "__giql_dj_cuts AS (" - f'SELECT t."{target_chrom}" AS kc, t."{target_start}" AS ks, ' - f't."{target_end}" AS ke, {t_start} AS pos FROM __giql_dj_tgt AS t ' - "UNION " - f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}", ' - f"{t_end} FROM __giql_dj_tgt AS t " - "UNION " - f'SELECT t."{target_chrom}", t."{target_start}", t."{target_end}", ' - "bp.pos FROM __giql_dj_tgt AS t JOIN __giql_dj_bp AS bp " - f"ON bp.chrom = {t_chrom} AND bp.pos > {t_start} " - f"AND bp.pos < {t_end})" - ) - segs_cte = ( - "__giql_dj_segs AS (" - "SELECT kc, ks, ke, pos AS seg_start, " - "LEAD(pos) OVER (PARTITION BY kc, ks, ke ORDER BY pos) AS seg_end " - "FROM __giql_dj_cuts)" - ) - # In self-reference mode the coverage EXISTS is provably always true: - # every emitted segment lies inside its parent target row, and that - # row is itself a member of the reference set. Skip the clause so the - # planner does not waste work on a no-op semi-join. The __giql_dj_ref - # CTE itself stays live because __giql_dj_bp still draws breakpoints - # from it. - where_clauses = ["s.seg_end IS NOT NULL", "s.seg_end > s.seg_start"] - if not is_self_reference: - where_clauses.append( - f'EXISTS (SELECT 1 FROM __giql_dj_ref AS r WHERE r."{ref_chrom}" = s.kc ' - f"AND {r_start} <= s.seg_start AND {r_end} > s.seg_start)" - ) - where_sql = " AND ".join(where_clauses) - final_select = ( - f"SELECT {passthrough}, s.kc AS disjoin_chrom, " - f"{out_start} AS disjoin_start, " - f"{out_end} AS disjoin_end FROM __giql_dj_tgt AS t " - f'JOIN __giql_dj_segs AS s ON t."{target_chrom}" = s.kc ' - f'AND t."{target_start}" = s.ks AND t."{target_end}" = s.ke ' - f"WHERE {where_sql}" - ) - return ( - f"(WITH {ref_cte}, {tgt_cte}, {bp_cte}, " - f"{cuts_cte}, {segs_cte} {final_select})" - ) - - def _disjoin_output_encoding( - self, expression: GIQLDisjoin, target_ref: ResolvedRef - ) -> Table | None: - """Return the target's declared encoding for DISJOIN's output round-trip. - - ``CanonicalizeCoordinates`` (pass 2) records the original - :class:`~giql.table.Table` on the resolution when it wraps a non-canonical - target (blanking the slot's own ``table``). For an unwrapped target — a - canonical registered table, or any target when the pass did not run — the - slot's own ``table`` carries the (identity) encoding. - - :param expression: - GIQLDisjoin expression node - :param target_ref: - The resolved target reference (post pass 2) - :return: - The target's declared :class:`~giql.table.Table`, or ``None`` - """ - resolution = expression.meta.get(META_KEY) - if isinstance(resolution, OperatorResolution): - preserved = resolution.output_tables.get("this") - if preserved is not None: - return preserved - return target_ref.table - - def _disjoin_passthrough( - self, target_start: str, target_end: str, output_table: Table | None - ) -> str: - """Project the target's full row, de-canonicalizing the interval columns. - - When the target's declared encoding is canonical 0-based half-open the - row passes through as a plain ``t.*`` — the byte-identical identity fast - path. When it is non-canonical the interval columns, canonical inside - ``__giql_dj_tgt``, are de-canonicalized back into that encoding via a star - ``REPLACE`` so the passed-through interval matches the target's own - convention. (Only non-canonical targets are wrapped, so the ``REPLACE`` - appears only where a canonical CTE already shapes the SQL.) - - :param target_start: - Physical start column name - :param target_end: - Physical end column name - :param output_table: - The target's declared :class:`~giql.table.Table`, or ``None`` - :return: - The passthrough projection fragment (``t.*`` or a star ``REPLACE``) - """ - if output_table is None or ( - output_table.coordinate_system == "0based" - and output_table.interval_type == "half_open" - ): - return "t.*" - pt_start = decanonical_start(f't."{target_start}"', output_table) - pt_end = decanonical_end(f't."{target_end}"', output_table) - return ( - f't.* REPLACE ({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' - ) - @staticmethod def _generate_distance_case( chrom_a: str, @@ -481,92 +310,6 @@ def _raise_nearest_reference_error( ) raise ValueError(f"Could not parse reference genomic range: {range_str}.") - def _disjoin_resolution( - self, expression: GIQLDisjoin - ) -> tuple[ResolvedRef, ResolvedRef, str]: - """Unpack the DISJOIN resolution attached by ResolveOperatorRefs (pass 1). - - Reads the :class:`~giql.resolver.OperatorResolution` from - ``expression.meta`` and returns ``(target_ref, ref, ref_from)`` where - ``ref_from`` is the text following ``FROM`` inside the ``__giql_dj_ref`` - CTE. A subquery reference carries no name, so it is rendered from the - AST node as an aliased derived table; registered tables and CTEs are - selected from by name. - - The resolver pass deliberately leaves unresolvable slots unresolved - (unregistered target; unsupported reference node type; reference name - using the reserved ``__giql_dj_`` prefix or matching neither a - registered table nor an in-query CTE). For those, and for a target name - using the reserved prefix (which the resolver does resolve), this - re-raises the generator's historical diagnostics verbatim. - - :param expression: - GIQLDisjoin expression node - :return: - Tuple of ``(target_ref, ref, ref_from)`` - :raises ValueError: - If the target or reference slot is unresolved, or a resolved name - uses the reserved ``__giql_dj_`` prefix. - """ - resolution = expression.meta.get(META_KEY) - target_ref = ( - resolution.slot("this") - if isinstance(resolution, OperatorResolution) - else None - ) - - # An unresolved target means it is not a registered table. - if target_ref is None: - target = expression.this - if isinstance(target, exp.Table): - target_name = target.name - elif isinstance(target, exp.Column): - target_name = target.table if target.table else str(target.this) - else: - target_name = str(target) - raise ValueError( - f"Target table '{target_name}' not found in tables. " - "Register the table before transpiling." - ) - - # The __giql_dj_ prefix names the operator's internal CTEs; a target - # table using it would collide with them. - if target_ref.name.startswith("__giql_dj_"): - raise ValueError( - f"DISJOIN target {target_ref.name!r} uses the reserved " - "'__giql_dj_' prefix, which names the operator's internal " - "CTEs. Rename the table." - ) - - reference = expression.args.get("reference") - ref = resolution.slot("reference") - if ref is not None: - if ref.kind == "subquery": - return target_ref, ref, f"{self.sql(reference)} AS __giql_dj_rs" - return target_ref, ref, ref.name - - # Unresolved reference: re-classify it and raise the matching - # historical diagnostic. An omitted reference always resolves (to the - # target set), so reference is non-None here. - if not isinstance(reference, (exp.Table, exp.Column, exp.Identifier)): - raise ValueError( - "DISJOIN reference must be a table name, a CTE, or a " - f"(SELECT ...) subquery; got {type(reference).__name__}: " - f"{reference}" - ) - ref_name = reference.name - if ref_name.startswith("__giql_dj_"): - raise ValueError( - f"DISJOIN reference {ref_name!r} uses the reserved " - "'__giql_dj_' prefix, which names the operator's internal " - "CTEs. Rename the reference relation." - ) - raise ValueError( - f"DISJOIN reference {ref_name!r} is neither a registered table " - "nor a CTE defined in this query. Register the table or define " - "the CTE before transpiling." - ) - @staticmethod def _extract_bool_param(param_expr: exp.Expression | None) -> bool: """Extract boolean value from a parameter expression. diff --git a/src/giql/resolver.py b/src/giql/resolver.py index 80d0969..7869d28 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -24,8 +24,8 @@ Scope note (epic #114, steps 1-3) --------------------------------- -The pass is behavior-preserving. DISJOIN's emitter -(``BaseGIQLGenerator.giqldisjoin_sql``, step 2) and NEAREST's emitter +The pass is behavior-preserving. DISJOIN's expander +(``giql.expanders.disjoin``, step 2) and NEAREST's emitter (``BaseGIQLGenerator.giqlnearest_sql``, step 3) consume the attached metadata; DISTANCE and the spatial predicates still use the generator's legacy resolver paths and ignore everything attached here until their port issues land. The @@ -73,6 +73,7 @@ from giql.constants import DEFAULT_END_COL from giql.constants import DEFAULT_START_COL from giql.constants import DEFAULT_STRAND_COL +from giql.constants import DJ_PREFIX from giql.expressions import Contains from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance @@ -758,7 +759,7 @@ def _resolve_disjoin_reference( ref_name = reference.name # The __giql_dj_ prefix names the operator's internal CTEs. - if ref_name.startswith("__giql_dj_"): + if ref_name.startswith(DJ_PREFIX): return None # A CTE from an enclosing WITH shadows a registered table of the same name diff --git a/src/giql/targets.py b/src/giql/targets.py index 825a6d2..88e4eb7 100644 --- a/src/giql/targets.py +++ b/src/giql/targets.py @@ -85,6 +85,15 @@ class GenericTarget(Target): This is the default target (``dialect=None``). Its capabilities are the conservative, maximally portable baseline that matches today's :class:`giql.generators.base.BaseGIQLGenerator` output. + + "SQL-92-ish", not strict SQL-92: because ``supports_star_replace=False``, the + DISJOIN passthrough over a **non-canonical** target falls back to a + ``SELECT * EXCEPT (...)`` projection (re-appending the de-canonicalized + interval columns). ``* EXCEPT`` is **not** SQL-92 and is **not + DuckDB-runnable** — it is a DataFusion-family extension — so the generic + target's non-canonical DISJOIN output runs only on an ``* EXCEPT``-capable + engine. A canonical (0-based half-open) target passes the row through as a + plain, fully portable ``SELECT *``. """ name: str = "generic" diff --git a/src/giql/transpile.py b/src/giql/transpile.py index b5568e2..3ff986a 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -230,7 +230,7 @@ def transpile( # Pass 1 of the normalization pipeline (epic #114): attach resolution # metadata to every GIQL operator slot ahead of generation. DISJOIN's - # emitter consumes this metadata (step 2); the remaining operators still + # expander consumes this metadata (step 2); the remaining operators still # use the generator's legacy resolver paths until their ports land. with _reraise_as_value_error("Resolution error"): ast = resolve_operator_refs(ast, tables_container) diff --git a/tests/integration/coordinate_space/conftest.py b/tests/integration/coordinate_space/conftest.py index 52f8d62..8b966d1 100644 --- a/tests/integration/coordinate_space/conftest.py +++ b/tests/integration/coordinate_space/conftest.py @@ -40,7 +40,10 @@ def giql_query(duckdb_connection): def _run(query: str, *, tables: list[Table], **table_data): for name, rows in table_data.items(): load_intervals(duckdb_connection, name, rows) - sql = transpile(query, tables=tables) + # Transpile for the DuckDB target since the SQL executes on DuckDB: a + # non-canonical DISJOIN passthrough emits DuckDB's ``* REPLACE`` here + # rather than the portable ``* EXCEPT`` form generic targets use. + sql = transpile(query, tables=tables, dialect="duckdb") return duckdb_connection.execute(sql).fetchall() return _run diff --git a/tests/integration/datafusion/test_cross_target_oracle.py b/tests/integration/datafusion/test_cross_target_oracle.py index f67ff9b..ba4a927 100644 --- a/tests/integration/datafusion/test_cross_target_oracle.py +++ b/tests/integration/datafusion/test_cross_target_oracle.py @@ -3,9 +3,9 @@ These exercise the reusable oracle (``tests/integration/conftest.py``) over the operators that already emit identical generic SQL across Generic and DataFusion and run correctly on DuckDB: INTERSECTS (literal + column-to-column join), -CONTAINS, WITHIN, and standalone NEAREST. The spatial predicates have since been -migrated to the expander registry (#141, epic #137); this lane locks in the -verification path that migration and every later one (#142-#144) consume. +CONTAINS, WITHIN, and standalone NEAREST. DISTANCE, the spatial predicates, +NEAREST, and DISJOIN have since been migrated to the expander registry (epic +#137); this lane locks in the verification path each migration consumes. For the non-join operators (DISTANCE, CONTAINS, WITHIN, ANY/ALL, CLUSTER, MERGE) the generic and datafusion targets emit byte-identical SQL and both run @@ -26,8 +26,10 @@ exposes the fallback's reserved ``__giql_x_*`` rank/key columns, a divergent output schema from the LATERAL form's — a known limitation pinned by the ``xfail`` ``test_correlated_nearest_star_projection_diverges_on_datafusion`` case -below and tracked for a query-level fix by #160 (dependent on #146). DISJOIN has -an analogous pending-#153 gap (duplicate ``end`` output names). +below and tracked for a query-level fix by #160 (dependent on #146). DISJOIN is +migrated onto the expander registry (#143) and its previously pending-#153 gap +(duplicate ``end`` output names) is closed, so its full three-target oracle now +runs as a real cross-target identity test. """ import pytest @@ -861,11 +863,12 @@ def test_merge_empty_input_returns_zero_rows(self, cross_target_oracle): class TestCrossTargetOracleDisjoin: - """DISJOIN: a DuckDB-only case plus the DataFusion duplicate-``end`` gap (#153). + """DISJOIN cross-target identity across Generic, DataFusion, and DuckDB (#143). - The DataFusion gap is the unaliased duplicate ``t."end"`` columns in the - ``__giql_dj_cuts`` CTE UNION branches, which DataFusion rejects as non-unique - projection names (DuckDB tolerates them). + DISJOIN is migrated onto the expander registry (#143), and the duplicate + unaliased ``t."end"`` columns in the ``__giql_dj_cuts`` CTE UNION branches are + aliased in every branch (#153), so DataFusion no longer rejects the projection + for non-unique names. The full three-target oracle therefore runs and agrees. """ def test_disjoin_splits_overlaps_on_duckdb(self, cross_target_oracle): @@ -893,40 +896,79 @@ def test_disjoin_splits_overlaps_on_duckdb(self, cross_target_oracle): targets=("duckdb",), ) - def test_disjoin_on_datafusion_unsupported_pending_153(self, cross_target_oracle): - """Test the full DISJOIN oracle raises DataFusion's duplicate-name error. + def test_disjoin_agrees_across_all_targets(self, cross_target_oracle): + """Test the full DISJOIN oracle returns identical rows on every target. Given: - The same two overlapping intervals. + Two overlapping intervals on chr1. + When: + The oracle runs all three targets — generic and datafusion execute the + registry-expanded DISJOIN (with every ``__giql_dj_cuts`` UNION branch + aliased, #153) on DataFusion, and duckdb runs on DuckDB. + Then: + Every target should return the same sub-segments, proving DISJOIN now + executes on DataFusion (the #153 duplicate-name gap is closed) and + agrees with DuckDB. + """ + # Arrange / Act / Assert + cross_target_oracle( + 'SELECT chrom, start, "end", disjoin_start, disjoin_end FROM DISJOIN(peaks)', + tables=[Table("peaks")], + peaks=[("chr1", 0, 100), ("chr1", 50, 150)], + expected=[ + ("chr1", 0, 100, 0, 50), + ("chr1", 0, 100, 50, 100), + ("chr1", 50, 150, 50, 100), + ("chr1", 50, 150, 100, 150), + ], + ) + + def test_disjoin_non_canonical_passthrough_agrees_across_targets( + self, cross_target_oracle + ): + """Test the non-canonical EXCEPT passthrough agrees with DuckDB REPLACE. + + Given: + Two overlapping intervals on chr1 in a table declared **1-based + closed** (non-canonical), each carrying an extra ``name`` passthrough + column. When: - The oracle runs all three targets — the datafusion target executes - DISJOIN's ``__giql_dj_cuts`` CTE, whose UNION branches project the - unaliased ``t."end"`` column twice. + The oracle runs all three targets — generic and datafusion exercise + the portable ``SELECT * EXCEPT (start, end), ...`` passthrough on + DataFusion (the #143 headline path, which runs on no engine without + this case), and duckdb runs the in-place ``* REPLACE`` form on DuckDB. Then: - DataFusion should reject the projection for non-unique expression - names (DuckDB tolerates it). This pins the known #153 gap: the - ``match`` narrows to the duplicate-output-name signature so an - unrelated/reworded DataFusion error fails loudly, and a closed gap - (no exception) trips pytest's "DID NOT RAISE", forcing this to be - converted into a real cross-target identity test when the duplicate - ``end`` columns in the ``__giql_dj_cuts`` UNION branches are aliased. + Every target should return the same rows: the de-canonicalized + (1-based closed) interval columns, the passthrough ``name``, and the + disjoin sub-segments. The projection lists **explicit** columns so the + row comparison is order-agnostic across the EXCEPT-re-append vs. + REPLACE-in-place column orders the two forms produce. """ # Arrange / Act / Assert - with pytest.raises( - Exception, match="Projections require unique expression names" - ): - cross_target_oracle( - 'SELECT chrom, start, "end", disjoin_start, disjoin_end ' - "FROM DISJOIN(peaks)", - tables=[Table("peaks")], - peaks=[("chr1", 0, 100), ("chr1", 50, 150)], - expected=[ - ("chr1", 0, 100, 0, 50), - ("chr1", 0, 100, 50, 100), - ("chr1", 50, 150, 50, 100), - ("chr1", 50, 150, 100, 150), - ], - ) + cross_target_oracle( + 'SELECT name, start, "end", disjoin_start, disjoin_end ' + "FROM DISJOIN(feats)", + tables=[ + Table( + "feats", + coordinate_system="1based", + interval_type="closed", + ) + ], + columns=( + ("chrom", "utf8"), + ("start", "int64"), + ("end", "int64"), + ("name", "utf8"), + ), + feats=[("chr1", 1, 100, "a"), ("chr1", 50, 150, "b")], + expected=[ + ("a", 1, 100, 1, 49), + ("a", 1, 100, 50, 100), + ("b", 50, 150, 50, 100), + ("b", 50, 150, 101, 150), + ], + ) class TestCrossTargetOracleDataShapes: diff --git a/tests/test_canonicalizer.py b/tests/test_canonicalizer.py index afea3e5..57b751c 100644 --- a/tests/test_canonicalizer.py +++ b/tests/test_canonicalizer.py @@ -21,6 +21,7 @@ from giql.canonicalizer import CANON_PREFIX from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect +from giql.expander import ExpandOperators from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance from giql.expressions import GIQLNearest @@ -30,6 +31,7 @@ from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables +from giql.targets import GenericTarget from giql.transpile import transpile hypothesis = pytest.importorskip("hypothesis") @@ -102,6 +104,10 @@ def test_canonical_table_sql_unchanged(self): query = "SELECT * FROM DISJOIN(variants)" tables = _tables(("0based", "half_open")) ast = resolve_operator_refs(parse_one(query, dialect=GIQLDialect), tables) + # Bypass pass 2 (canonicalization) but keep pass 3 (expansion): DISJOIN + # now goes through its registered expander, so the pass-bypassed reference + # must too, isolating pass 2's contribution to nothing. + ast = ExpandOperators(GenericTarget(), tables).transform(ast) expected = BaseGIQLGenerator(tables=tables).generate(ast) # Act @@ -130,6 +136,10 @@ def test_non_canonical_table_sql_unchanged(self, monkeypatch): tables = Tables() tables.register("variants", variants) ast = resolve_operator_refs(parse_one(query, dialect=GIQLDialect), tables) + # Bypass pass 2 (canonicalization) but keep pass 3 (expansion): DISJOIN + # now expands through its registry entry, so the pass-bypassed reference + # must run pass 3 too for the byte-identical comparison to isolate pass 2. + ast = ExpandOperators(GenericTarget(), tables).transform(ast) expected = BaseGIQLGenerator(tables=tables).generate(ast) # Act diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index 34412b9..426adf0 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -32,6 +32,29 @@ def test_giqldisjoin_sql_should_emit_with_cte_subquery(self): assert "UNION" in sql assert "LEAD(" in sql + def test_transpile_should_alias_every_cuts_union_branch(self): + """Test that every __giql_dj_cuts UNION branch aliases its columns (#153). + + Given: + A default (canonical 0-based half-open) DISJOIN, whose ``end`` + de-canonicalizes to the bare physical ``t."end"`` column. + When: + Transpiling to SQL. + Then: + Each of the three ``__giql_dj_cuts`` UNION branches should project a + uniquely aliased ``ke`` (start/end/breakpoint), and the unaliased + ``t."end", t."end" FROM`` collision shape the #153 fix removed should + be absent — so DataFusion accepts the projection's unique names. + """ + # Arrange & act + sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) + + # Assert: every cuts branch aliases the end-column projection as ``ke``. + assert sql.count("AS kc") == 3 + assert sql.count("AS ke") == 3 + # The pre-#153 unaliased duplicate-end collision must not survive. + assert 't."end", t."end" FROM' not in sql + def test_giqldisjoin_sql_should_emit_disjoin_columns(self): """Test that DISJOIN appends the sub-interval columns. @@ -309,7 +332,7 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_reference_omitted(self): sql = transpile("SELECT * FROM DISJOIN(features)", tables=["features"]) # Assert - assert "EXISTS (" not in sql + assert "EXISTS(" not in sql def test_giqldisjoin_sql_should_skip_exists_clause_when_reference_resolves_to_target( self, @@ -331,7 +354,7 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_reference_resolves_to_ta ) # Assert - assert "EXISTS (" not in sql + assert "EXISTS(" not in sql def test_giqldisjoin_sql_should_emit_exists_clause_when_reference_is_different_table( self, @@ -354,7 +377,7 @@ def test_giqldisjoin_sql_should_emit_exists_clause_when_reference_is_different_t ) # Assert - assert "EXISTS (" in sql + assert "EXISTS(" in sql def test_giqldisjoin_sql_should_emit_exists_clause_when_cte_shadows_target_name( self, @@ -377,7 +400,7 @@ def test_giqldisjoin_sql_should_emit_exists_clause_when_cte_shadows_target_name( ) # Assert - assert "EXISTS (" in sql + assert "EXISTS(" in sql def test_giqldisjoin_sql_should_emit_exists_clause_when_reference_is_subquery(self): """Test that a subquery reference keeps the EXISTS clause. @@ -397,7 +420,7 @@ def test_giqldisjoin_sql_should_emit_exists_clause_when_reference_is_subquery(se ) # Assert - assert "EXISTS (" in sql + assert "EXISTS(" in sql def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_one_based_closed_encoding( self, @@ -425,7 +448,7 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_one_based_cl ) # Assert - assert "EXISTS (" not in sql + assert "EXISTS(" not in sql assert '("start" - 1) AS "start"' in sql def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_custom_column_names( @@ -450,7 +473,7 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_target_uses_custom_colum ) # Assert - assert "EXISTS (" not in sql + assert "EXISTS(" not in sql assert 'r."seqid"' not in sql assert 'r."lo"' not in sql assert 'r."hi"' not in sql @@ -483,7 +506,7 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_explicit_self_reference_ ) # Assert - assert "EXISTS (" not in sql + assert "EXISTS(" not in sql assert '("start" - 1) AS "start"' in sql # Target and explicit self-reference dedupe to a single wrapper CTE: one # CTE definition plus the two FROM references (target + reference). @@ -501,7 +524,7 @@ def test_giqldisjoin_sql_should_emit_one_exists_clause_when_query_mixes_self_and When: Transpiling to SQL. Then: - It should produce SQL containing exactly one ``EXISTS (`` + It should produce SQL containing exactly one ``EXISTS(`` occurrence — only the distinct-reference call retains the coverage filter. """ @@ -514,7 +537,7 @@ def test_giqldisjoin_sql_should_emit_one_exists_clause_when_query_mixes_self_and ) # Assert - assert sql.count("EXISTS (") == 1 + assert sql.count("EXISTS(") == 1 @pytest.mark.parametrize( ("query", "tables", "skips_exists"), @@ -571,7 +594,7 @@ def test_giqldisjoin_sql_should_pin_coverage_skippable_across_reference_shapes( sql = transpile(query, tables=tables) # Assert - assert ("EXISTS (" not in sql) is skips_exists + assert ("EXISTS(" not in sql) is skips_exists def test_giqldisjoin_sql_should_emit_exists_clause_when_enclosing_cte_shadows_target_from_outer_scope( self, @@ -599,7 +622,7 @@ def test_giqldisjoin_sql_should_emit_exists_clause_when_enclosing_cte_shadows_ta ) # Assert - assert sql.count("EXISTS (") == 1 + assert sql.count("EXISTS(") == 1 def test_giqldisjoin_sql_should_resolve_recursive_cte_reference(self): """Test that a WITH RECURSIVE CTE reference resolves to the CTE. @@ -627,7 +650,7 @@ def test_giqldisjoin_sql_should_resolve_recursive_cte_reference(self): # Assert assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql - assert "EXISTS (" in sql + assert "EXISTS(" in sql assert '("start" - 1)' not in sql def test_giqldisjoin_sql_should_resolve_set_operation_attached_cte_reference(self): @@ -656,7 +679,7 @@ def test_giqldisjoin_sql_should_resolve_set_operation_attached_cte_reference(sel # Assert assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql - assert sql.count("EXISTS (") == 1 + assert sql.count("EXISTS(") == 1 def test_giqldisjoin_sql_should_resolve_redeclared_inner_cte_reference(self): """Test that an inner WITH redeclaring an outer CTE name still resolves. @@ -683,7 +706,7 @@ def test_giqldisjoin_sql_should_resolve_redeclared_inner_cte_reference(self): # Assert assert "__giql_dj_ref AS (SELECT * FROM refs)" in sql - assert sql.count("EXISTS (") == 1 + assert sql.count("EXISTS(") == 1 def test_giqldisjoin_sql_should_not_see_cte_declared_in_sibling_derived_table( self, @@ -810,17 +833,50 @@ def test_giqldisjoin_sql_should_dedupe_self_reference_to_one_canon_cte(self): assert sql.count("AS (SELECT * REPLACE") == 1 assert sql.count("__giql_canon_") == 3 - def test_giqldisjoin_sql_should_emit_passthrough_interval_in_target_encoding(self): - """Test that the passed-through interval is de-canonicalized to the target. + def test_transpile_should_emit_portable_passthrough_interval_on_generic_target( + self, + ): + """Test that the passthrough uses a portable EXCEPT projection by default. + + Given: + A self-mode DISJOIN over a 1-based closed target whose row passes + through canonical inside the wrapper CTE, transpiled for the generic + target (which does not support ``SELECT * REPLACE``). + When: + Transpiling to SQL. + Then: + It should de-canonicalize the passed-through start/end back into the + target's encoding via a portable ``* EXCEPT`` projection that drops + the interval columns and re-projects them recomputed. + """ + # Arrange & act + sql = transpile( + "SELECT * FROM DISJOIN(features)", + tables=[ + Table("features", coordinate_system="1based", interval_type="closed") + ], + ) + + # Assert + assert ( + 't.* EXCEPT ("start", "end"), (t."start" + 1) AS "start", ' + 't."end" AS "end"' in sql + ) + + def test_transpile_should_emit_star_replace_passthrough_on_duckdb_target( + self, + ): + """Test that the passthrough uses ``* REPLACE`` on a REPLACE-capable target. Given: A self-mode DISJOIN over a 1-based closed target whose row passes - through canonical inside the wrapper CTE. + through canonical inside the wrapper CTE, transpiled for the DuckDB + target (which supports ``SELECT * REPLACE``). When: Transpiling to SQL. Then: It should de-canonicalize the passed-through start/end back into the - target's encoding via a star REPLACE on the final projection. + target's encoding via a star ``REPLACE`` on the final projection. """ # Arrange & act sql = transpile( @@ -828,6 +884,7 @@ def test_giqldisjoin_sql_should_emit_passthrough_interval_in_target_encoding(sel tables=[ Table("features", coordinate_system="1based", interval_type="closed") ], + dialect="duckdb", ) # Assert diff --git a/tests/test_usage_patterns.py b/tests/test_usage_patterns.py index c82dca7..7b3dd29 100644 --- a/tests/test_usage_patterns.py +++ b/tests/test_usage_patterns.py @@ -89,9 +89,12 @@ def _execute(usage: OperatorUsage, query: str, engine: str) -> list: The descriptor's `tables` yields a `Table` config for any fixture with a custom physical layout, so GIQL resolves custom column names and - coordinate systems during transpilation. + coordinate systems during transpilation. The query is transpiled for the + engine's own target dialect so engine-specific emission (e.g. DISJOIN's + capability-driven non-canonical passthrough, which uses ``* REPLACE`` on + DuckDB) executes on the engine it is shaped for. """ - sql = transpile(query, tables=list(usage.tables)) + sql = transpile(query, tables=list(usage.tables), dialect=engine) rows = ENGINES[engine](usage.fixtures, sql) return sorted(([list(row) for row in rows]), key=repr) From 3950b192726f522b03eaf7702e4c49337b515279 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 29 Jun 2026 22:40:23 -0400 Subject: [PATCH 097/142] refactor: Relocate CLUSTER and MERGE into the operator-expander registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Migrate the last two operators off the pre-pass transformer chain and onto the ExpandOperators registry (epic #137), completing the operator migration so every GIQL operator dispatches through one mechanism. CLUSTER and MERGE are whole-query rewrites, so each expander navigates to its enclosing SELECT and mutates it in place, returning the operator node unchanged (a no-op replace) — the same pattern NEAREST's decorrelated fallback uses, required because the canonical form puts the operator at the root SELECT, which has no parent to replace through. The pass's deepest-first walk subsumes the transformers' manual recursion into CTEs and subqueries. Columns are derived from the FROM table, so the operators carry an empty resolution and deliberately skip canonicalization. Emitted SQL is semantically identical to the legacy output and now deterministic; the only intended byte difference is the injected lag_calc column order, which the legacy output left hash-seed-dependent. Because the in-place rewrite copies the enclosing WHERE/HAVING into the new inner subquery, the expander re-runs the pass over the restructured SELECT (the active registry is threaded through ExpansionContext) so a sibling operator carried into that subquery — a spatial predicate or DISTANCE in a filtered cluster/merge — is expanded rather than leaking to the generator. The standalone ClusterTransformer and MergeTransformer classes are removed and the pre-pass calls are dropped from the transpile pipeline. Some shapes that the legacy pipeline accepted only to emit non-executable or invalid SQL now raise a clear ValueError instead: CLUSTER and MERGE in a single SELECT (in-place mutation cannot express both), multiple CLUSTER expressions in one SELECT (as multiple MERGE already did), and a CLUSTER or MERGE nested inside a larger projection expression. The MERGE GROUP BY chromosome term is now quoted like every other column reference, so a reserved-word custom chrom column emits valid SQL. No working query is affected. Synthesized identifiers that do not yet use the reserved __giql_ prefix are left as-is for a follow-up; renaming them would change emitted SQL. --- docs/dialect/aggregation-operators.rst | 9 + src/giql/expander.py | 12 +- src/giql/expanders/cluster.py | 564 +++++++++++++++++++++ src/giql/expanders/merge.py | 245 +++++++++ src/giql/expressions.py | 35 +- src/giql/resolver.py | 36 +- src/giql/transformer.py | 661 +------------------------ src/giql/transpile.py | 15 +- tests/expanders/test_cluster.py | 370 ++++++++++++++ tests/expanders/test_merge.py | 292 +++++++++++ tests/test_expander.py | 191 ++++--- tests/test_transpile.py | 6 + 12 files changed, 1671 insertions(+), 765 deletions(-) create mode 100644 src/giql/expanders/cluster.py create mode 100644 src/giql/expanders/merge.py create mode 100644 tests/expanders/test_cluster.py create mode 100644 tests/expanders/test_merge.py diff --git a/docs/dialect/aggregation-operators.rst b/docs/dialect/aggregation-operators.rst index f34d1c0..69d7f65 100644 --- a/docs/dialect/aggregation-operators.rst +++ b/docs/dialect/aggregation-operators.rst @@ -401,6 +401,15 @@ Notes - MERGE is an aggregate operation that processes all matching rows - The operation sorts data internally, so pre-sorting is not required +.. note:: + + CLUSTER and MERGE cannot be combined in a single ``SELECT`` — MERGE + aggregates rows away while CLUSTER is a per-row window over those same rows, + so no single query expresses both. Transpiling ``SELECT MERGE(interval), + CLUSTER(interval) FROM features`` raises a ``ValueError``. Use them in + separate queries instead — for example, CLUSTER over a subquery, or MERGE + over one. + Related Operators ~~~~~~~~~~~~~~~~~ diff --git a/src/giql/expander.py b/src/giql/expander.py index ae9ca3e..bab00fb 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -102,9 +102,15 @@ class ExpansionContext: ``capabilities`` and ``sqlglot_dialect``. tables : Tables The registered :class:`~giql.table.Tables` container. + registry : ExpanderRegistry | None + The registry the pass is resolving against. Carried so a whole-query + rewrite (CLUSTER / MERGE) can re-enter :func:`expand_operators` over the + SELECT it just restructured and expand sibling operators it copied into + it, honoring a custom-registry pass run. ``None`` for a standalone + context built outside the pass. """ - __slots__ = ("node", "resolution", "target", "tables", "_alias_seq") + __slots__ = ("node", "resolution", "target", "tables", "registry", "_alias_seq") def __init__( self, @@ -113,11 +119,13 @@ def __init__( target: Target, tables: Tables, alias_seq: Callable[[], str] | None = None, + registry: ExpanderRegistry | None = None, ) -> None: self.node = node self.resolution = resolution self.target = target self.tables = tables + self.registry = registry # A single sequence is threaded across every context built for one # ``ExpandOperators`` run so aliases minted for sibling operators never # collide; a standalone context falls back to its own sequence. @@ -514,7 +522,7 @@ class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for "valid resolution metadata; pass 1 (resolve_operator_refs) must " "run first and annotate every operator node." ) - ctx = ExpansionContext(node, resolution, target, tables, alias_seq) + ctx = ExpansionContext(node, resolution, target, tables, alias_seq, registry=reg) replacement = fn(node, ctx) if not isinstance(replacement, exp.Expression): raise TypeError( diff --git a/src/giql/expanders/cluster.py b/src/giql/expanders/cluster.py new file mode 100644 index 0000000..6285f82 --- /dev/null +++ b/src/giql/expanders/cluster.py @@ -0,0 +1,564 @@ +"""The CLUSTER operator expander (epic #137, issue #144). + +CLUSTER assigns a cluster id to every interval, grouping each run of mutually +adjacent intervals (optionally within a maximum gap, strand-aware, and gated by a +pairwise predicate) under one id. It cannot be a single window function because it +needs a window over a window (``SUM`` over a ``LAG``-derived flag), so the +expansion restructures the enclosing query into a two-level form:: + + SELECT *, CLUSTER(interval) AS cluster_id FROM features + +becomes:: + + SELECT *, SUM(is_new_cluster) OVER (PARTITION BY chrom ORDER BY start) AS cluster_id + FROM ( + SELECT *, + CASE WHEN LAG(end) OVER (...) >= start THEN 0 ELSE 1 END AS is_new_cluster + FROM features + ) AS lag_calc + +(The ``becomes::`` form is simplified for readability; the emitted SQL quotes +identifiers and appends ``NULLS LAST`` to the window ORDER BY.) + +This module is the AST-expansion replacement for the legacy +:class:`giql.transformer.ClusterTransformer`, which ran as a pre-pass transformer +on the raw parsed AST. It produces the same SQL; the existing CLUSTER +transpilation and bedtools oracle tests are the migration oracle. + +Unlike the node-local expanders for DISTANCE / NEAREST / DISJOIN — whose result +replaces a single node — CLUSTER is a **whole-query rewrite**. It shares only the +*shape* of :func:`giql.expanders.nearest._fallback_form` — return the operator +node unchanged so the pass's ``node.replace`` is a no-op — but the *mechanism* +differs. ``nearest`` does an in-place child ``.replace`` because its LATERAL has a +parent to replace through; CLUSTER instead copies the enclosing +:class:`~sqlglot.expressions.Select`, restructures the copy, and *transplants* its +contents back onto the original SELECT (see :func:`transplant`). That root- +preserving transplant is required because the canonical +``SELECT *, CLUSTER(...) FROM t`` puts CLUSTER at the *root* ``SELECT``, which has +no parent to replace it through; transplanting preserves the root's identity (the +pass returns the same expression object it was handed). + +The pass walks the tree and collects every operator node, expanding deepest-first, +so this expander never recurses into CTEs / subqueries: a nested CLUSTER is +collected and expanded on its own. The shared cluster-restructure helper +(:func:`expand_cluster_query`) is reused by :mod:`giql.expanders.merge`, since +MERGE is built on CLUSTER. +""" + +from __future__ import annotations + +from typing import NamedTuple +from typing import TypeVar + +from sqlglot import exp + +from giql.constants import DEFAULT_CHROM_COL +from giql.constants import DEFAULT_END_COL +from giql.constants import DEFAULT_START_COL +from giql.constants import DEFAULT_STRAND_COL +from giql.expander import ExpansionContext +from giql.expander import expand_operators +from giql.expander import register +from giql.expressions import GIQLCluster +from giql.expressions import GIQLMerge +from giql.table import Tables +from giql.targets import GenericTarget + +_T = TypeVar("_T", bound=exp.Expression) + + +class GenomicColumns(NamedTuple): + """The resolved physical column names CLUSTER / MERGE operate over. + + Derived from the enclosing FROM table by :func:`genomic_columns`. A + :class:`~typing.NamedTuple` so it still unpacks and indexes positionally + (``chrom, start, end, strand = columns``) while giving the four fields names. + """ + + chrom: str + start: str + end: str + strand: str + + +@register(GenericTarget, GIQLCluster) +def expand_cluster(node: GIQLCluster, ctx: ExpansionContext) -> exp.Expression: + """Expand a CLUSTER node by restructuring its enclosing SELECT in place. + + Registered for :class:`~giql.targets.GenericTarget`, so every target resolves + to it through the registry's generic chain (CLUSTER emits identical SQL across + targets). Locates the enclosing :class:`~sqlglot.expressions.Select`, derives + the genomic columns from the FROM table, and rewrites that SELECT in place into + the two-level ``lag_calc`` form. Returns the original CLUSTER node (now + unreachable from the rewritten root SELECT) so the pass's ``node.replace`` is a + no-op. + + Parameters + ---------- + node : GIQLCluster + The CLUSTER node being expanded. + ctx : ExpansionContext + The expansion context; only its :attr:`~ExpansionContext.tables` is read + (CLUSTER derives its columns from the enclosing FROM table, not from + resolution metadata). + + Returns + ------- + exp.Expression + The CLUSTER node, unchanged — the surrounding SELECT was mutated in place. + """ + select = node.find_ancestor(exp.Select) + if select is None: + # Defensive: CLUSTER only ever parses inside a SELECT projection. With no + # enclosing SELECT there is nothing to restructure; leave the node for the + # generator to error on, exactly as the legacy transformer's + # non-Select guard did. + return node + require_top_level_projection(select, node, GIQLCluster) + reject_cluster_merge_mix(select) + columns = genomic_columns(select, ctx.tables) + # Build the transformed query from a detached copy so the intermediate never + # aliases live nodes, then transplant its args onto the original SELECT to + # preserve its identity (and so a root SELECT is rewritten without a parent to + # replace it through). + source = select.copy() + transformed = expand_cluster_query(source, columns) + if transformed is None: + # No-op for a CLUSTER that parses outside the SELECT projection (e.g. in + # WHERE / ORDER BY): find_projected finds none in the copy, so there is + # nothing to restructure and the operator leaks to the generator exactly + # as on `main`. require_top_level_projection above already rejects the + # in-projection-expression case; this guards the out-of-projection case. + return node + transplant(select, transformed) + # copy()+transplant duplicated the enclosing WHERE / HAVING into the new + # lag_calc subquery; the originals the pass collected are now unreachable, so + # re-run the pass over the restructured SELECT to expand any sibling pass-3 + # operators (spatial predicates, DISTANCE) carried into it. Safe from + # recursion: the CLUSTER node is already replaced by its SUM window. (#144 B1) + expand_operators(select, ctx.target, ctx.tables, ctx.registry) + return node + + +def genomic_columns(select: exp.Select, tables: Tables) -> GenomicColumns: + """Return the ``(chrom, start, end, strand)`` columns for *select*'s FROM table. + + Part of the shared CLUSTER/MERGE expansion toolkit (reused by + :mod:`giql.expanders.merge`). Mirrors the legacy + ``ClusterTransformer._get_genomic_columns`` / ``_get_table_name``: read the + FROM-clause table name, look it up in *tables*, and use its configured column + names, falling back to the canonical defaults (and to the default strand + column when the table declares none). + """ + table_name: str | None = None + from_clause = select.args.get("from_") + if from_clause is not None and isinstance(from_clause.this, exp.Table): + table_name = from_clause.this.name + + chrom_col = DEFAULT_CHROM_COL + start_col = DEFAULT_START_COL + end_col = DEFAULT_END_COL + strand_col = DEFAULT_STRAND_COL + + if table_name: + table = tables.get(table_name) + if table: + chrom_col = table.chrom_col + start_col = table.start_col + end_col = table.end_col + if table.strand_col: + strand_col = table.strand_col + + return GenomicColumns(chrom_col, start_col, end_col, strand_col) + + +def extract_stranded(stranded_expr: exp.Expression | None) -> bool: + """Coerce a CLUSTER/MERGE ``stranded`` operand to a bool. Shared toolkit. + + Mirrors the legacy per-transformer coercion exactly: a missing operand is + ``False``; an ``exp.Boolean`` yields its raw ``.this``; an ``exp.Literal`` + compares case-folded to ``TRUE``. The final two arms (``exp.Literal`` and the + string-truthiness fallback) are defensive — the GIQL grammar only ever produces + ``exp.Boolean`` for ``stranded := `` — retained for parity with the + legacy port. + """ + if stranded_expr is None: + return False + if isinstance(stranded_expr, exp.Boolean): + return stranded_expr.this + if isinstance(stranded_expr, exp.Literal): + return str(stranded_expr.this).upper() == "TRUE" + return str(stranded_expr).upper() in ("TRUE", "1", "YES") + + +def expand_cluster_query( + query: exp.Select, columns: GenomicColumns +) -> exp.Select | None: + """Restructure *query* for every CLUSTER in its projection; ``None`` if none. + + Finds CLUSTER expressions in *query*'s SELECT list and rewrites the query into + the two-level ``lag_calc`` form once per CLUSTER (chaining, as the legacy + transformer did). Operates on *query* directly and returns the rewritten + query. Returns ``None`` when *query* projects no CLUSTER, so callers can treat + that as a no-op. + + Reused by :mod:`giql.expanders.merge`, whose intermediate clustered query is + restructured through this same helper. + """ + cluster_exprs = find_projected(query, GIQLCluster) + if not cluster_exprs: + return None + if len(cluster_exprs) > 1: + # Chaining the rewrite per CLUSTER yields a duplicate ``lag_calc`` alias + # and an ``is_new_cluster`` binder error — non-executable SQL. Fail loudly, + # mirroring the multiple-MERGE guard, rather than emitting it (#144 A15). + raise ValueError("Multiple CLUSTER expressions not yet supported") + for cluster_expr in cluster_exprs: + query = _transform_for_cluster(query, cluster_expr, columns) + return query + + +def find_projected(select: exp.Select, op_type: type[_T]) -> list[_T]: + """Return *select*'s projected operators of *op_type* (bare or aliased). Toolkit. + + Shared CLUSTER/MERGE primitive: both expanders and the co-occurrence guard + locate their operator the same way — a top-level SELECT projection item that + either *is* the operator or is an ``exp.Alias`` wrapping it. + """ + found: list[_T] = [] + for expression in select.expressions: + if isinstance(expression, op_type): + found.append(expression) + elif isinstance(expression, exp.Alias) and isinstance( + expression.this, op_type + ): + found.append(expression.this) + return found + + +def require_top_level_projection( + select: exp.Select, node: exp.Expression, op_type: type +) -> None: + """Raise if *node* is buried inside a projection expression. Shared toolkit. + + A CLUSTER / MERGE is only expandable as a *top-level* projection item — bare + or directly aliased — because the whole-query rewrite restructures the SELECT + around it. One nested inside a larger projection expression such as + ``ABS(CLUSTER(interval))`` has no coherent rewrite and would otherwise leak an + unexpanded operator to the generator, so fail loudly here (#144 A16). An + operator that parses *outside* the projection entirely (e.g. in WHERE / + ORDER BY) is not under any projection item and is left for the expander's + existing no-op path. + """ + operator = op_type.__name__.removeprefix("GIQL").upper() + for projection in select.expressions: + inner = projection.this if isinstance(projection, exp.Alias) else projection + if inner is node: + return + if any(descendant is node for descendant in inner.walk()): + raise ValueError( + f"{operator} must be a top-level projection item; it cannot be " + "nested inside another expression (e.g. a function call or " + "arithmetic)." + ) + + +def reject_cluster_merge_mix(select: exp.Select) -> None: + """Raise if *select* projects both a CLUSTER and a MERGE. Shared toolkit. + + The two are mutually incompatible in one SELECT: MERGE aggregates the rows + away (it rewrites the query into a ``GROUP BY`` over a clustered subquery) + while CLUSTER is a per-row window over those same rows, so no coherent single + query expresses both. The legacy pre-pass chained the two transformers and + emitted *non-executable* SQL for this shape (a window over ``GROUP BY``- + aggregated rows — a DuckDB ``BinderException``, never a leaked operator). The + new in-place ``transplant`` cannot express both at all: whichever expander + runs first rewrites the shared SELECT and strands the sibling as an unexpanded + node. Fail loudly here — mirroring the ``Multiple MERGE expressions not yet + supported`` guard — so the combination raises a clear diagnostic rather than + emitting broken SQL. + """ + if find_projected(select, GIQLCluster) and find_projected(select, GIQLMerge): + raise ValueError( + "CLUSTER and MERGE cannot be combined in a single SELECT; MERGE " + "aggregates rows while CLUSTER is a per-row window. Use them in " + "separate queries (e.g. CLUSTER over a subquery, or MERGE over one)." + ) + + +def _transform_for_cluster( + query: exp.Select, cluster_expr: GIQLCluster, columns: GenomicColumns +) -> exp.Select: + """Rewrite *query* into the two-level ``lag_calc`` form for one CLUSTER. + + A byte-for-byte port of the legacy + ``ClusterTransformer._transform_for_cluster``, with the genomic columns passed + in (the legacy method re-derived them from the FROM table) rather than read off + ``self``. Builds an inner ``lag_calc`` subquery that materializes an + ``is_new_cluster`` flag from a ``LAG`` window, then an outer query whose + ``SUM(is_new_cluster) OVER (...)`` window replaces the CLUSTER projection. + """ + chrom_col, start_col, end_col, strand_col = columns + + # Extract CLUSTER parameters + distance_expr = cluster_expr.args.get("distance") + + # Handle distance parameter - could be int literal or None + if distance_expr: + if isinstance(distance_expr, exp.Literal): + distance = int(distance_expr.this) + else: + # Defensive: the grammar only yields exp.Literal for a distance + # argument, so this non-Literal fallback is unreachable via the parser; + # retained for parity with the legacy port. + try: + distance = int(str(distance_expr.this)) + except (ValueError, AttributeError): + distance = 0 + else: + distance = 0 + + stranded = extract_stranded(cluster_expr.args.get("stranded")) + + # Build partition clause + partition_cols = [exp.column(chrom_col, quoted=True)] + if stranded: + partition_cols.append(exp.column(strand_col, quoted=True)) + + # Build ORDER BY for window + order_by = [exp.Ordered(this=exp.column(start_col, quoted=True))] + + # Create LAG window spec + lag_window = exp.Window( + this=exp.Anonymous(this="LAG", expressions=[exp.column(end_col, quoted=True)]), + partition_by=partition_cols, + order=exp.Order(expressions=order_by), + ) + + # Add distance offset if specified + if distance > 0: + lag_with_distance = exp.Add( + this=lag_window, expression=exp.Literal.number(distance) + ) + else: + lag_with_distance = lag_window + + # Build the adjacency condition (predecessor end >= current start). + adjacency = exp.GTE( + this=lag_with_distance, + expression=exp.column(start_col, quoted=True), + ) + + # An optional predicate further restricts which adjacent intervals + # are kept together: a row stays in the current cluster only when it + # is adjacent to its predecessor AND the predicate holds between them. + # ``PREV(col)`` references in the predicate resolve to the predecessor + # row via LAG over the same partition/order as the adjacency window. + predicate_expr = cluster_expr.args.get("predicate") + if predicate_expr is not None: + rewritten_predicate = _rewrite_predecessor_refs( + predicate_expr, partition_cols, order_by + ) + keep_together = exp.And( + this=adjacency, + expression=exp.Paren(this=rewritten_predicate), + ) + else: + keep_together = adjacency + + # Create CASE expression for is_new_cluster + case_expr = exp.Case( + ifs=[ + exp.If( + this=keep_together, + true=exp.Literal.number(0), + ) + ], + default=exp.Literal.number(1), + ) + + # Build CTE SELECT expressions (all original except CLUSTER, plus is_new_cluster) + cte_expressions = [] + for expression in query.expressions: + # Skip CLUSTER expressions + if isinstance(expression, GIQLCluster): + continue + elif isinstance(expression, exp.Alias) and isinstance( + expression.this, GIQLCluster + ): + continue + else: + cte_expressions.append(expression) + + # Ensure required columns for window functions are included + required_cols = {chrom_col, start_col, end_col} + if stranded: + required_cols.add(strand_col) + + # The predicate is evaluated inside the lag_calc CTE, so every column + # it references (current-row columns and PREV() arguments alike) must + # be projected into that CTE. Folding them into required_cols makes the + # scope dependency explicit and keeps the columns available even when a + # later operator wraps this query in a further subquery. + if predicate_expr is not None: + required_cols |= {col.name for col in predicate_expr.find_all(exp.Column)} + + # Check if required columns are already in the select list + selected_cols = set() + for expr in cte_expressions: + if isinstance(expr, exp.Column): + selected_cols.add(expr.name) + elif isinstance(expr, exp.Alias): + # Don't count aliases as the source column + pass + elif isinstance(expr, exp.Star): + # SELECT * includes all columns + selected_cols = required_cols # Assume all are covered + break + + # Add missing required columns + # Sort the residual so the injected-column order is deterministic across runs + # (set-difference iteration order is PYTHONHASHSEED-dependent; the legacy port + # left it nondeterministic — see review #144 A2). + for col in sorted(required_cols - selected_cols): + cte_expressions.append(exp.column(col, quoted=True)) + + # Add is_new_cluster calculation + # NOTE: synthesized name; the missing __giql_ reserved prefix is left to #161. + cte_expressions.append(exp.alias_(case_expr, "is_new_cluster", quoted=False)) + + # Build CTE query + cte_select = exp.Select() + cte_select.select(*cte_expressions, copy=False) + + # Copy FROM, WHERE, GROUP BY, HAVING from original (but not ORDER BY) + # Use copy() to avoid sharing references between queries + if query.args.get("from_"): + from_clause = query.args["from_"].copy() + cte_select.set("from_", from_clause) + if query.args.get("where"): + cte_select.set("where", query.args["where"].copy()) + if query.args.get("group"): + cte_select.set("group", query.args["group"].copy()) + if query.args.get("having"): + cte_select.set("having", query.args["having"].copy()) + + # Create outer query with SUM over is_new_cluster + sum_window = exp.Window( + this=exp.Sum(this=exp.column("is_new_cluster")), + partition_by=partition_cols, + order=exp.Order(expressions=order_by), + ) + + # Build outer SELECT expressions (replace CLUSTER with SUM) + new_expressions = [] + for expression in query.expressions: + if isinstance(expression, GIQLCluster): + new_expressions.append(sum_window) + elif isinstance(expression, exp.Alias) and isinstance( + expression.this, GIQLCluster + ): + # Keep the alias but replace the expression + new_expressions.append( + exp.alias_(sum_window, expression.alias, quoted=False) + ) + else: + new_expressions.append(expression) + + # Build new query + new_query = exp.Select() + new_query.select(*new_expressions, copy=False) + + # Wrap CTE in subquery and set as FROM clause + # NOTE: synthesized name; the missing __giql_ reserved prefix is left to #161. + subquery = exp.Subquery( + this=cte_select, + alias=exp.TableAlias(this=exp.Identifier(this="lag_calc")), + ) + new_query.from_(subquery, copy=False) + + # Copy ORDER BY from original to outer query + if query.args.get("order"): + new_query.order_by(*query.args["order"].expressions, copy=False) + + return new_query + + +def _rewrite_predecessor_refs( + predicate: exp.Expression, + partition_cols: list[exp.Expression], + order_by: list[exp.Ordered], +) -> exp.Expression: + """Rewrite ``PREV(col)`` calls in a predicate to LAG windows. + + A byte-for-byte port of the legacy + ``ClusterTransformer._rewrite_predecessor_refs``. Bare column references in the + predicate denote the current interval. Each ``PREV(col)`` call denotes the + sorted predecessor's value of that column and is rewritten to ``LAG(col) OVER + (...)`` using the same partition/order as the cluster's adjacency window, so + the predicate is evaluated pairwise against the immediately preceding row. + Every column identifier (current-row columns and LAG arguments alike) is quoted + so that reserved-word genomic columns such as ``start`` / ``end`` are emitted as + valid SQL. + + :param predicate: + Boolean predicate expression to rewrite (not mutated). + :param partition_cols: + Window partition columns (chromosome, optionally strand). + :param order_by: + Window ORDER BY terms (start position). + :return: + A copy of the predicate with every ``PREV(...)`` call replaced by an + equivalent LAG window and all column identifiers quoted. + :raises ValueError: + If a ``PREV()`` call does not take exactly one argument, or if a + ``PREV()`` call is nested inside another (predicates compare only the + immediate predecessor). + """ + + def _is_prev(node: exp.Expression) -> bool: + return isinstance(node, exp.Anonymous) and node.name.upper() == "PREV" + + def _replace(node: exp.Expression) -> exp.Expression: + if _is_prev(node): + args = node.expressions + if len(args) != 1: + raise ValueError( + f"PREV() takes exactly one column argument; got {len(args)}." + ) + if any(_is_prev(inner) for inner in args[0].find_all(exp.Anonymous)): + raise ValueError( + "PREV() cannot be nested; a CLUSTER/MERGE predicate " + "compares only the immediate predecessor." + ) + return exp.Window( + this=exp.Anonymous(this="LAG", expressions=[args[0].copy()]), + partition_by=[col.copy() for col in partition_cols], + order=exp.Order(expressions=[term.copy() for term in order_by]), + ) + return node + + rewritten = predicate.copy().transform(_replace) + for column in rewritten.find_all(exp.Column): + column.this.set("quoted", True) + return rewritten + + +def transplant(select: exp.Select, new: exp.Select) -> None: + """Replace *select*'s contents with *new*'s, preserving *select*'s identity. + + Part of the shared CLUSTER/MERGE expansion toolkit. Clears every argument of + *select* and re-installs *new*'s, so *select* keeps its position in the + surrounding tree (and its identity as the object the pass returns) while taking + on the rewritten structure. This is how a whole-query rewrite is applied to a + *root* SELECT, which has no parent to ``replace`` through. + + Precondition: *new* MUST be a detached throwaway — a freshly-built + ``exp.Select``, as the ``_transform_for_*`` helpers return. Its children are + re-parented onto *select*, so passing a node still attached elsewhere would + corrupt that other tree. + """ + assert new.parent is None, "transplant() requires a detached `new` subtree" + select.args.clear() + for key, value in list(new.args.items()): + select.set(key, value) diff --git a/src/giql/expanders/merge.py b/src/giql/expanders/merge.py new file mode 100644 index 0000000..4191635 --- /dev/null +++ b/src/giql/expanders/merge.py @@ -0,0 +1,245 @@ +"""The MERGE operator expander (epic #137, issue #144). + +MERGE combines overlapping (and, with parameters, adjacent / strand-matched / +predicate-gated) intervals into single intervals. It is built on CLUSTER: assign +a cluster id, then aggregate ``MIN(start)`` / ``MAX(end)`` per cluster:: + + SELECT MERGE(interval) FROM features + +becomes:: + + SELECT chrom, MIN(start) AS start, MAX(end) AS end + FROM (SELECT *, CLUSTER(interval) AS __giql_cluster_id FROM features) AS clustered + GROUP BY chrom, __giql_cluster_id + ORDER BY chrom, start + +(The ``becomes::`` form is simplified for readability; the emitted SQL quotes +identifiers, appends ``NULLS LAST``, and the inner ``CLUSTER(...)`` is itself +expanded into the two-level ``lag_calc`` form.) + +This module is the AST-expansion replacement for the legacy +:class:`giql.transformer.MergeTransformer`; it produces the same SQL (the existing +MERGE transpilation and bedtools oracle tests are the migration oracle). + +Like CLUSTER (:mod:`giql.expanders.cluster`), MERGE is a **whole-query rewrite**: +it navigates to the enclosing :class:`~sqlglot.expressions.Select`, mutates it in +place, and returns the operator node unchanged so the pass's ``node.replace`` is a +no-op. The intermediate clustered subquery is restructured through CLUSTER's shared +:func:`giql.expanders.cluster.expand_cluster_query`, mirroring how the legacy +``MergeTransformer`` composed ``ClusterTransformer``. +""" + +from __future__ import annotations + +from sqlglot import exp + +from giql.expander import ExpansionContext +from giql.expander import expand_operators +from giql.expander import register + +# Shared CLUSTER/MERGE expansion toolkit (MERGE is built on CLUSTER). +from giql.expanders.cluster import GenomicColumns +from giql.expanders.cluster import expand_cluster_query +from giql.expanders.cluster import extract_stranded +from giql.expanders.cluster import find_projected +from giql.expanders.cluster import genomic_columns +from giql.expanders.cluster import reject_cluster_merge_mix +from giql.expanders.cluster import require_top_level_projection +from giql.expanders.cluster import transplant +from giql.expressions import GIQLCluster +from giql.expressions import GIQLMerge +from giql.targets import GenericTarget + + +@register(GenericTarget, GIQLMerge) +def expand_merge(node: GIQLMerge, ctx: ExpansionContext) -> exp.Expression: + """Expand a MERGE node by restructuring its enclosing SELECT in place. + + Registered for :class:`~giql.targets.GenericTarget` (MERGE emits identical SQL + across targets). Locates the enclosing + :class:`~sqlglot.expressions.Select`, derives the genomic columns from the FROM + table, and rewrites that SELECT in place into the clustered-aggregation form. + Returns the original MERGE node (now unreachable from the rewritten root SELECT) + so the pass's ``node.replace`` is a no-op. + + Parameters + ---------- + node : GIQLMerge + The MERGE node being expanded. + ctx : ExpansionContext + The expansion context; only its :attr:`~ExpansionContext.tables` is read. + + Returns + ------- + exp.Expression + The MERGE node, unchanged — the surrounding SELECT was mutated in place. + """ + select = node.find_ancestor(exp.Select) + if select is None: + return node + require_top_level_projection(select, node, GIQLMerge) + reject_cluster_merge_mix(select) + columns = genomic_columns(select, ctx.tables) + # Build from a detached copy, then transplant onto the original SELECT to + # preserve its identity (and rewrite a root SELECT without a parent to replace + # through), exactly as the CLUSTER expander does. + source = select.copy() + transformed = _transform_select_merge(source, columns) + if transformed is None: + # No-op for a MERGE that parses outside the SELECT projection (e.g. in + # WHERE / ORDER BY): find_projected finds none in the copy, so there is + # nothing to restructure and the operator leaks to the generator exactly + # as on `main`. require_top_level_projection above already rejects the + # in-projection-expression case; this guards the out-of-projection case. + return node + transplant(select, transformed) + # copy()+transplant duplicated the enclosing WHERE into the new clustered + # subquery; the originals the pass collected are now unreachable, so re-run the + # pass over the restructured SELECT to expand any sibling pass-3 operators + # carried into it. Safe from recursion: the MERGE is already gone. (#144 B1) + expand_operators(select, ctx.target, ctx.tables, ctx.registry) + return node + + +def _transform_select_merge( + query: exp.Select, columns: GenomicColumns +) -> exp.Select | None: + """Rewrite *query* for the MERGE in its projection; ``None`` if none. + + Mirrors the legacy ``MergeTransformer.transform`` dispatch: find the MERGE + expressions, reject more than one, and delegate the single supported case to + :func:`_transform_for_merge`. + """ + merge_exprs = find_projected(query, GIQLMerge) + if not merge_exprs: + return None + # For now, support only one MERGE expression + if len(merge_exprs) > 1: + raise ValueError("Multiple MERGE expressions not yet supported") + return _transform_for_merge(query, merge_exprs[0], columns) + + +def _transform_for_merge( + query: exp.Select, merge_expr: GIQLMerge, columns: GenomicColumns +) -> exp.Select: + """Rewrite *query* into the clustered-aggregation form for one MERGE. + + A byte-for-byte port of the legacy ``MergeTransformer._transform_for_merge``, + with the genomic columns passed in and the intermediate clustered query + restructured through CLUSTER's shared + :func:`giql.expanders.cluster.expand_cluster_query` (the legacy method called + ``ClusterTransformer.transform``). Builds an inner ``clustered`` subquery that + appends ``__giql_cluster_id``, then an outer query that aggregates + ``MIN(start)`` / ``MAX(end)`` per cluster. + """ + chrom_col, start_col, end_col, strand_col = columns + + # Extract MERGE parameters (same as CLUSTER) + distance_expr = merge_expr.args.get("distance") + stranded_expr = merge_expr.args.get("stranded") + predicate_expr = merge_expr.args.get("predicate") + + # Build CLUSTER expression with same parameters + cluster_kwargs = {"this": merge_expr.this} + if distance_expr: + cluster_kwargs["distance"] = distance_expr + if stranded_expr: + cluster_kwargs["stranded"] = stranded_expr + if predicate_expr is not None: + cluster_kwargs["predicate"] = predicate_expr + + cluster_expr = GIQLCluster(**cluster_kwargs) + + # Create intermediate query with CLUSTER + # Start with original query's FROM/WHERE/etc + cluster_query = exp.Select() + cluster_query.select(exp.Star(), copy=False) + # NOTE: __giql_cluster_id carries the reserved prefix; the un-prefixed + # `clustered` / `lag_calc` / `is_new_cluster` siblings are left to #161. + cluster_query.select( + exp.alias_(cluster_expr, "__giql_cluster_id", quoted=False), + append=True, + copy=False, + ) + + # Copy FROM, WHERE from original + # Use copy() to avoid sharing references between queries + if query.args.get("from_"): + cluster_query.set("from_", query.args["from_"].copy()) + if query.args.get("where"): + cluster_query.set("where", query.args["where"].copy()) + + # Apply CLUSTER transformation to get the CTE-based query + clustered = expand_cluster_query(cluster_query, columns) + assert clustered is not None, "intermediate MERGE cluster query has no CLUSTER" + cluster_query = clustered + + # Build GROUP BY columns + # Quote the chrom column (as every other chrom reference is) so a reserved-word + # custom chrom column emits valid SQL (#144 A13). + group_by_cols = [exp.column(chrom_col, quoted=True)] + + # Handle stranded parameter + stranded = extract_stranded(stranded_expr) + + if stranded: + group_by_cols.append(exp.column(strand_col, quoted=True)) + + group_by_cols.append(exp.column("__giql_cluster_id")) + + # Build SELECT expressions for merged intervals + select_exprs = [] + + # Add group-by columns (non-aggregated) + select_exprs.append(exp.column(chrom_col, quoted=True)) + if stranded: + select_exprs.append(exp.column(strand_col, quoted=True)) + + # Add merged interval bounds + select_exprs.append( + exp.alias_( + exp.Min(this=exp.column(start_col, quoted=True)), start_col, quoted=False + ) + ) + select_exprs.append( + exp.alias_( + exp.Max(this=exp.column(end_col, quoted=True)), end_col, quoted=False + ) + ) + + # Process other columns from original SELECT + for expression in query.expressions: + # Skip the MERGE expression itself + if isinstance(expression, GIQLMerge): + continue + elif isinstance(expression, exp.Alias) and isinstance( + expression.this, GIQLMerge + ): + continue + # Include other columns (they should be aggregates or in GROUP BY) + else: + select_exprs.append(expression) + + # Build final query + final_query = exp.Select() + final_query.select(*select_exprs, copy=False) + + # FROM the clustered subquery + subquery = exp.Subquery( + this=cluster_query, + alias=exp.TableAlias(this=exp.Identifier(this="clustered")), + ) + final_query.from_(subquery, copy=False) + + # Add GROUP BY + final_query.group_by(*group_by_cols, copy=False) + + # Add ORDER BY (chromosome, start) + final_query.order_by( + exp.Ordered(this=exp.column(chrom_col, quoted=True)), copy=False + ) + final_query.order_by( + exp.Ordered(this=exp.column(start_col, quoted=True)), append=True, copy=False + ) + + return final_query diff --git a/src/giql/expressions.py b/src/giql/expressions.py index 6a0f440..ffbf017 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -110,15 +110,6 @@ class SpatialPredicate(exp.Binary): #: half-open) operands are left untouched and the emitted SQL stays byte-identical. _CANONICALIZE = True -#: Default opt-out from the ``ExpandOperators`` pass (epic #137, step 2). The -#: per-operator ``GIQL_EXPAND`` flag mirrors ``GIQL_CANONICALIZE``: an operator -#: takes the new AST-expansion path only when it sets ``GIQL_EXPAND = True`` *and* -#: an expander is registered for it; otherwise the legacy ``*_sql`` emitter runs. -#: This is the opt-out default: an operator inherits it and stays on the legacy -#: emitter until its migration step flips the flag to ``True`` alongside its -#: registered expander. Operators already migrated override it on their own class. -_EXPAND = False - class Intersects(SpatialPredicate): """INTERSECTS spatial predicate. @@ -244,11 +235,14 @@ class GIQLCluster(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } - # Inert today: the CLUSTER/MERGE transformers rewrite these nodes before the - # ExpandOperators pass runs, so the pass never sees a GIQLCluster to dispatch - # and this flag is not a live opt-in. It is forward-looking for #144, which - # migrates these operators onto the expander registry. - GIQL_EXPAND = _EXPAND + #: Migrated to the ExpandOperators pass (epic #137, issue #144). CLUSTER is + #: expanded by ``giql.expanders.cluster`` — a whole-query rewrite into the + #: two-level ``lag_calc`` form — replacing the legacy + #: ``giql.transformer.ClusterTransformer`` pre-pass transformer. Note CLUSTER + #: deliberately does NOT set ``GIQL_CANONICALIZE``: the expander derives its + #: columns from the FROM table, so pass 2 is intentionally a no-op here and the + #: emitted SQL stays byte-identical to the legacy pre-pass output. + GIQL_EXPAND = True @classmethod def from_arg_list(cls, args): @@ -291,11 +285,14 @@ class GIQLMerge(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } - # Inert today: the CLUSTER/MERGE transformers rewrite these nodes before the - # ExpandOperators pass runs, so the pass never sees a GIQLMerge to dispatch - # and this flag is not a live opt-in. It is forward-looking for #144, which - # migrates these operators onto the expander registry. - GIQL_EXPAND = _EXPAND + #: Migrated to the ExpandOperators pass (epic #137, issue #144). MERGE is + #: expanded by ``giql.expanders.merge`` — a whole-query rewrite into the + #: clustered-aggregation form (built on CLUSTER) — replacing the legacy + #: ``giql.transformer.MergeTransformer`` pre-pass transformer. Like CLUSTER, + #: MERGE deliberately does NOT set ``GIQL_CANONICALIZE`` (columns come from the + #: FROM table), so pass 2 is intentionally a no-op and the SQL stays + #: byte-identical to the legacy pre-pass output. + GIQL_EXPAND = True @classmethod def from_arg_list(cls, args): diff --git a/src/giql/resolver.py b/src/giql/resolver.py index 7869d28..8431255 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -22,18 +22,20 @@ asserts every operator slot carries well-formed resolution metadata, mirroring ``sqlglot``'s ``validate_qualify_columns`` and Spark's ``CheckAnalysis``. -Scope note (epic #114, steps 1-3) ---------------------------------- -The pass is behavior-preserving. DISJOIN's expander -(``giql.expanders.disjoin``, step 2) and NEAREST's emitter -(``BaseGIQLGenerator.giqlnearest_sql``, step 3) consume the attached metadata; -DISTANCE and the spatial predicates still use the generator's legacy resolver -paths and ignore everything attached here until their port issues land. The -resolution semantics computed here mirror the generator's historical -``_resolve_target_table`` / ``_resolve_disjoin_reference`` / -``_enclosing_cte_names`` (DISJOIN) and ``_resolve_nearest_reference`` / -``_find_outer_table_in_lateral_join`` (NEAREST) behavior exactly; all of those -helpers now live only here. +Scope note (epic #114 / #137) +----------------------------- +The pass is behavior-preserving. Every operator in the ``_OPERATORS`` roster now +takes the ``ExpandOperators`` path (epic #137): the DISJOIN, NEAREST, DISTANCE, +and spatial-predicate (INTERSECTS / CONTAINS / WITHIN / ``SpatialSetPredicate``) +expanders in :mod:`giql.expanders` consume the metadata attached here, resolving +their reference slots and column operands through it. CLUSTER and MERGE (#144) +declare no reference slots, so they resolve to an empty-but-valid +:class:`OperatorResolution` and their expanders derive columns from the enclosing +FROM table rather than from resolution. The resolution semantics computed here +mirror the generator's historical ``_resolve_target_table`` / +``_resolve_disjoin_reference`` / ``_enclosing_cte_names`` (DISJOIN) and +``_resolve_nearest_reference`` / ``_find_outer_table_in_lateral_join`` (NEAREST) +behavior exactly; all of those helpers now live only here. Two consequences of the zero-behavior-change constraint shape the implementation: @@ -75,8 +77,10 @@ from giql.constants import DEFAULT_STRAND_COL from giql.constants import DJ_PREFIX from giql.expressions import Contains +from giql.expressions import GIQLCluster from giql.expressions import GIQLDisjoin from giql.expressions import GIQLDistance +from giql.expressions import GIQLMerge from giql.expressions import GIQLNearest from giql.expressions import Intersects from giql.expressions import SlotSpec @@ -128,11 +132,17 @@ DEFAULT_END_COL, ) -#: The GIQL operator expression classes the pass inspects. +#: The GIQL operator expression classes the pass inspects. CLUSTER and MERGE +#: (#144) declare no reference slots, so they resolve to an empty-but-valid +#: ``OperatorResolution``; the ``ExpandOperators`` pass requires that metadata to +#: be present, and their expanders derive columns from the enclosing FROM table +#: rather than from resolution. _OPERATORS: tuple[type[exp.Expression], ...] = ( GIQLDisjoin, GIQLNearest, GIQLDistance, + GIQLCluster, + GIQLMerge, Intersects, Contains, Within, diff --git a/src/giql/transformer.py b/src/giql/transformer.py index c757aaf..e75d254 100644 --- a/src/giql/transformer.py +++ b/src/giql/transformer.py @@ -1,8 +1,9 @@ """Query transformers for GIQL operations. -This module contains transformers that rewrite queries containing GIQL-specific -operations (like CLUSTER, MERGE, and binned INTERSECTS joins) into equivalent -SQL with CTEs. +This module contains the pre-pass transformers that rewrite column-to-column +INTERSECTS joins (the binned equi-join and DuckDB IEJoin plans) into equivalent +SQL with CTEs. CLUSTER and MERGE were relocated to the operator-expander registry +(``giql.expanders.cluster`` / ``giql.expanders.merge``) in epic #137 (#144). """ from dataclasses import dataclass @@ -18,9 +19,6 @@ from giql.constants import DEFAULT_CHROM_COL from giql.constants import DEFAULT_END_COL from giql.constants import DEFAULT_START_COL -from giql.constants import DEFAULT_STRAND_COL -from giql.expressions import GIQLCluster -from giql.expressions import GIQLMerge from giql.expressions import Intersects from giql.table import Table from giql.table import Tables @@ -228,657 +226,6 @@ def from_intersects( ) -class ClusterTransformer: - """Transforms queries containing CLUSTER into CTE-based queries. - - CLUSTER cannot be a simple window function because it requires nested - window functions (LAG inside SUM). Instead, we transform: - - SELECT *, CLUSTER(interval) AS cluster_id FROM features - - Into: - - WITH lag_calc AS ( - SELECT *, LAG(end_pos) OVER (...) AS prev_end FROM features - ) - SELECT *, SUM(CASE WHEN prev_end >= start_pos ...) AS cluster_id - FROM lag_calc - """ - - def __init__(self, tables: Tables): - """Initialize transformer. - - :param tables: - Table configurations for column mapping - """ - self.tables = tables - - def _get_table_name(self, query: exp.Select) -> str | None: - """Extract table name from query's FROM clause. - - :param query: - Query to extract table name from - :return: - Table name if FROM contains a simple table, None otherwise - """ - from_clause = query.args.get("from_") - if not from_clause: - return None - - if isinstance(from_clause.this, exp.Table): - return from_clause.this.name - - return None - - def _get_genomic_columns(self, query: exp.Select) -> tuple[str, str, str, str]: - """Get genomic column names from table config or defaults. - - :param query: - Query to extract table and column info from - :return: - Tuple of (chrom_col, start_col, end_col, strand_col) - """ - table_name = self._get_table_name(query) - - # Default column names - chrom_col = DEFAULT_CHROM_COL - start_col = DEFAULT_START_COL - end_col = DEFAULT_END_COL - strand_col = DEFAULT_STRAND_COL - - if table_name: - table = self.tables.get(table_name) - if table: - chrom_col = table.chrom_col - start_col = table.start_col - end_col = table.end_col - if table.strand_col: - strand_col = table.strand_col - - return chrom_col, start_col, end_col, strand_col - - def transform(self, query: exp.Expression) -> exp.Expression: - """Transform query if it contains CLUSTER expressions. - - :param query: - Parsed query AST - :return: - Transformed query AST - """ - if not isinstance(query, exp.Select): - return query - - # First, recursively transform any CTEs that might contain CLUSTER - if query.args.get("with_"): - cte = query.args["with_"] - for cte_expr in cte.expressions: - if isinstance(cte_expr, exp.CTE): - # Transform the CTE's subquery - cte_expr.set("this", self.transform(cte_expr.this)) - - # Recursively transform subqueries in FROM clause - if query.args.get("from_"): - from_clause = query.args["from_"] - self._transform_subqueries_in_node(from_clause) - - # Recursively transform subqueries in JOIN clauses - if query.args.get("joins"): - for join in query.args["joins"]: - self._transform_subqueries_in_node(join) - - # Recursively transform subqueries in WHERE clause - if query.args.get("where"): - self._transform_subqueries_in_node(query.args["where"]) - - # Find all CLUSTER expressions in the SELECT clause - cluster_exprs = self._find_cluster_expressions(query) - - if not cluster_exprs: - return query - - # Transform query for each CLUSTER expression - for cluster_expr in cluster_exprs: - query = self._transform_for_cluster(query, cluster_expr) - - return query - - def _transform_subqueries_in_node(self, node: exp.Expression): - """Recursively transform subqueries within an expression node. - - :param node: - Expression node to search for subqueries - """ - # Find and transform any Subquery nodes - for subquery in node.find_all(exp.Subquery): - if isinstance(subquery.this, exp.Select): - transformed = self.transform(subquery.this) - subquery.set("this", transformed) - - def _find_cluster_expressions(self, query: exp.Select) -> list[GIQLCluster]: - """Find all CLUSTER expressions in query. - - :param query: - Query to search - :return: - List of CLUSTER expressions - """ - cluster_exprs = [] - - for expression in query.expressions: - # Check if this is a CLUSTER expression or an alias containing one - if isinstance(expression, GIQLCluster): - cluster_exprs.append(expression) - elif isinstance(expression, exp.Alias): - if isinstance(expression.this, GIQLCluster): - cluster_exprs.append(expression.this) - - return cluster_exprs - - def _transform_for_cluster( - self, query: exp.Select, cluster_expr: GIQLCluster - ) -> exp.Select: - """Transform query to compute CLUSTER using CTEs. - - :param query: - Original query - :param cluster_expr: - CLUSTER expression to transform - :return: - Transformed query with CTEs - """ - # Extract CLUSTER parameters - distance_expr = cluster_expr.args.get("distance") - - # Handle distance parameter - could be int literal or None - if distance_expr: - if isinstance(distance_expr, exp.Literal): - distance = int(distance_expr.this) - else: - # Try to extract as string and convert - try: - distance = int(str(distance_expr.this)) - except (ValueError, AttributeError): - distance = 0 - else: - distance = 0 - - stranded_expr = cluster_expr.args.get("stranded") - if stranded_expr: - # Handle different types of boolean expressions - if isinstance(stranded_expr, exp.Boolean): - stranded = stranded_expr.this - elif isinstance(stranded_expr, exp.Literal): - stranded = str(stranded_expr.this).upper() == "TRUE" - else: - # Try to extract the value as a string - stranded = str(stranded_expr).upper() in ("TRUE", "1", "YES") - else: - stranded = False - - # Get column names from table config or use defaults - chrom_col, start_col, end_col, strand_col = self._get_genomic_columns(query) - - # Build partition clause - partition_cols = [exp.column(chrom_col, quoted=True)] - if stranded: - partition_cols.append(exp.column(strand_col, quoted=True)) - - # Build ORDER BY for window - order_by = [exp.Ordered(this=exp.column(start_col, quoted=True))] - - # Create LAG window spec - lag_window = exp.Window( - this=exp.Anonymous( - this="LAG", expressions=[exp.column(end_col, quoted=True)] - ), - partition_by=partition_cols, - order=exp.Order(expressions=order_by), - ) - - # Add distance offset if specified - if distance > 0: - lag_with_distance = exp.Add( - this=lag_window, expression=exp.Literal.number(distance) - ) - else: - lag_with_distance = lag_window - - # Build the adjacency condition (predecessor end >= current start). - adjacency = exp.GTE( - this=lag_with_distance, - expression=exp.column(start_col, quoted=True), - ) - - # An optional predicate further restricts which adjacent intervals - # are kept together: a row stays in the current cluster only when it - # is adjacent to its predecessor AND the predicate holds between them. - # ``PREV(col)`` references in the predicate resolve to the predecessor - # row via LAG over the same partition/order as the adjacency window. - predicate_expr = cluster_expr.args.get("predicate") - if predicate_expr is not None: - rewritten_predicate = self._rewrite_predecessor_refs( - predicate_expr, partition_cols, order_by - ) - keep_together = exp.And( - this=adjacency, - expression=exp.Paren(this=rewritten_predicate), - ) - else: - keep_together = adjacency - - # Create CASE expression for is_new_cluster - case_expr = exp.Case( - ifs=[ - exp.If( - this=keep_together, - true=exp.Literal.number(0), - ) - ], - default=exp.Literal.number(1), - ) - - # Build CTE SELECT expressions (all original except CLUSTER, plus is_new_cluster) - cte_expressions = [] - for expression in query.expressions: - # Skip CLUSTER expressions - if isinstance(expression, GIQLCluster): - continue - elif isinstance(expression, exp.Alias) and isinstance( - expression.this, GIQLCluster - ): - continue - else: - cte_expressions.append(expression) - - # Ensure required columns for window functions are included - required_cols = {chrom_col, start_col, end_col} - if stranded: - required_cols.add(strand_col) - - # The predicate is evaluated inside the lag_calc CTE, so every column - # it references (current-row columns and PREV() arguments alike) must - # be projected into that CTE. Folding them into required_cols makes the - # scope dependency explicit and keeps the columns available even when a - # later operator wraps this query in a further subquery. - if predicate_expr is not None: - required_cols |= {col.name for col in predicate_expr.find_all(exp.Column)} - - # Check if required columns are already in the select list - selected_cols = set() - for expr in cte_expressions: - if isinstance(expr, exp.Column): - selected_cols.add(expr.name) - elif isinstance(expr, exp.Alias): - # Don't count aliases as the source column - pass - elif isinstance(expr, exp.Star): - # SELECT * includes all columns - selected_cols = required_cols # Assume all are covered - break - - # Add missing required columns - for col in required_cols - selected_cols: - cte_expressions.append(exp.column(col, quoted=True)) - - # Add is_new_cluster calculation - cte_expressions.append(exp.alias_(case_expr, "is_new_cluster", quoted=False)) - - # Build CTE query - cte_select = exp.Select() - cte_select.select(*cte_expressions, copy=False) - - # Copy FROM, WHERE, GROUP BY, HAVING from original (but not ORDER BY) - # Use copy() to avoid sharing references between queries - if query.args.get("from_"): - from_clause = query.args["from_"].copy() - cte_select.set("from_", from_clause) - if query.args.get("where"): - cte_select.set("where", query.args["where"].copy()) - if query.args.get("group"): - cte_select.set("group", query.args["group"].copy()) - if query.args.get("having"): - cte_select.set("having", query.args["having"].copy()) - - # Create outer query with SUM over is_new_cluster - sum_window = exp.Window( - this=exp.Sum(this=exp.column("is_new_cluster")), - partition_by=partition_cols, - order=exp.Order(expressions=order_by), - ) - - # Build outer SELECT expressions (replace CLUSTER with SUM) - new_expressions = [] - for expression in query.expressions: - if isinstance(expression, GIQLCluster): - new_expressions.append(sum_window) - elif isinstance(expression, exp.Alias) and isinstance( - expression.this, GIQLCluster - ): - # Keep the alias but replace the expression - new_expressions.append( - exp.alias_(sum_window, expression.alias, quoted=False) - ) - else: - new_expressions.append(expression) - - # Build new query - new_query = exp.Select() - new_query.select(*new_expressions, copy=False) - - # Wrap CTE in subquery and set as FROM clause - subquery = exp.Subquery( - this=cte_select, - alias=exp.TableAlias(this=exp.Identifier(this="lag_calc")), - ) - new_query.from_(subquery, copy=False) - - # Copy ORDER BY from original to outer query - if query.args.get("order"): - new_query.order_by(*query.args["order"].expressions, copy=False) - - return new_query - - def _rewrite_predecessor_refs( - self, - predicate: exp.Expression, - partition_cols: list[exp.Expression], - order_by: list[exp.Ordered], - ) -> exp.Expression: - """Rewrite ``PREV(col)`` calls in a predicate to LAG windows. - - Bare column references in the predicate denote the current interval. - Each ``PREV(col)`` call denotes the sorted predecessor's value of that - column and is rewritten to ``LAG(col) OVER (...)`` using the same - partition/order as the cluster's adjacency window, so the predicate is - evaluated pairwise against the immediately preceding row. Every column - identifier (current-row columns and LAG arguments alike) is quoted so - that reserved-word genomic columns such as ``start`` / ``end`` are - emitted as valid SQL, matching how the rest of this transformer quotes - genomic columns. - - :param predicate: - Boolean predicate expression to rewrite (not mutated). - :param partition_cols: - Window partition columns (chromosome, optionally strand). - :param order_by: - Window ORDER BY terms (start position). - :return: - A copy of the predicate with every ``PREV(...)`` call replaced by an - equivalent LAG window and all column identifiers quoted. - :raises ValueError: - If a ``PREV()`` call does not take exactly one argument, or if a - ``PREV()`` call is nested inside another (predicates compare only - the immediate predecessor). - """ - - def _is_prev(node: exp.Expression) -> bool: - return isinstance(node, exp.Anonymous) and node.name.upper() == "PREV" - - def _replace(node: exp.Expression) -> exp.Expression: - if _is_prev(node): - args = node.expressions - if len(args) != 1: - raise ValueError( - f"PREV() takes exactly one column argument; got {len(args)}." - ) - if any(_is_prev(inner) for inner in args[0].find_all(exp.Anonymous)): - raise ValueError( - "PREV() cannot be nested; a CLUSTER/MERGE predicate " - "compares only the immediate predecessor." - ) - return exp.Window( - this=exp.Anonymous(this="LAG", expressions=[args[0].copy()]), - partition_by=[col.copy() for col in partition_cols], - order=exp.Order(expressions=[term.copy() for term in order_by]), - ) - return node - - rewritten = predicate.copy().transform(_replace) - for column in rewritten.find_all(exp.Column): - column.this.set("quoted", True) - return rewritten - - -class MergeTransformer: - """Transforms queries containing MERGE into GROUP BY queries. - - MERGE combines overlapping intervals using CLUSTER + aggregation: - - SELECT MERGE(interval) FROM features - - Into: - - WITH clustered AS ( - SELECT *, CLUSTER(interval) AS __giql_cluster_id FROM features - ) - SELECT - chromosome, - MIN(start_pos) AS start_pos, - MAX(end_pos) AS end_pos - FROM clustered - GROUP BY chromosome, __giql_cluster_id - ORDER BY chromosome, start_pos - """ - - def __init__(self, tables: Tables): - """Initialize transformer. - - :param tables: - Table configurations for column mapping - """ - self.tables = tables - self.cluster_transformer = ClusterTransformer(tables) - - def transform(self, query: exp.Expression) -> exp.Expression: - """Transform query if it contains MERGE expressions. - - :param query: - Parsed query AST - :return: - Transformed query AST - """ - if not isinstance(query, exp.Select): - return query - - # First, recursively transform any CTEs that might contain MERGE - if query.args.get("with_"): - cte = query.args["with_"] - for cte_expr in cte.expressions: - if isinstance(cte_expr, exp.CTE): - # Transform the CTE's subquery - cte_expr.set("this", self.transform(cte_expr.this)) - - # Recursively transform subqueries in FROM clause - if query.args.get("from_"): - from_clause = query.args["from_"] - self._transform_subqueries_in_node(from_clause) - - # Recursively transform subqueries in JOIN clauses - if query.args.get("joins"): - for join in query.args["joins"]: - self._transform_subqueries_in_node(join) - - # Recursively transform subqueries in WHERE clause - if query.args.get("where"): - self._transform_subqueries_in_node(query.args["where"]) - - # Find all MERGE expressions in the SELECT clause - merge_exprs = self._find_merge_expressions(query) - - if not merge_exprs: - return query - - # For now, support only one MERGE expression - if len(merge_exprs) > 1: - raise ValueError("Multiple MERGE expressions not yet supported") - - merge_expr = merge_exprs[0] - return self._transform_for_merge(query, merge_expr) - - def _transform_subqueries_in_node(self, node: exp.Expression): - """Recursively transform subqueries within an expression node. - - :param node: - Expression node to search for subqueries - """ - # Find and transform any Subquery nodes - for subquery in node.find_all(exp.Subquery): - if isinstance(subquery.this, exp.Select): - transformed = self.transform(subquery.this) - subquery.set("this", transformed) - - def _find_merge_expressions(self, query: exp.Select) -> list[GIQLMerge]: - """Find all MERGE expressions in query. - - :param query: - Query to search - :return: - List of MERGE expressions - """ - merge_exprs = [] - - for expression in query.expressions: - if isinstance(expression, GIQLMerge): - merge_exprs.append(expression) - elif isinstance(expression, exp.Alias): - if isinstance(expression.this, GIQLMerge): - merge_exprs.append(expression.this) - - return merge_exprs - - def _transform_for_merge( - self, query: exp.Select, merge_expr: GIQLMerge - ) -> exp.Select: - """Transform query to compute MERGE using CLUSTER + GROUP BY. - - :param query: - Original query - :param merge_expr: - MERGE expression to transform - :return: - Transformed query with clustering and aggregation - """ - # Extract MERGE parameters (same as CLUSTER) - distance_expr = merge_expr.args.get("distance") - stranded_expr = merge_expr.args.get("stranded") - predicate_expr = merge_expr.args.get("predicate") - - # Get column names from table config or use defaults - ( - chrom_col, - start_col, - end_col, - strand_col, - ) = self.cluster_transformer._get_genomic_columns(query) - - # Build CLUSTER expression with same parameters - cluster_kwargs = {"this": merge_expr.this} - if distance_expr: - cluster_kwargs["distance"] = distance_expr - if stranded_expr: - cluster_kwargs["stranded"] = stranded_expr - if predicate_expr is not None: - cluster_kwargs["predicate"] = predicate_expr - - cluster_expr = GIQLCluster(**cluster_kwargs) - - # Create intermediate query with CLUSTER - # Start with original query's FROM/WHERE/etc - cluster_query = exp.Select() - cluster_query.select(exp.Star(), copy=False) - cluster_query.select( - exp.alias_(cluster_expr, "__giql_cluster_id", quoted=False), - append=True, - copy=False, - ) - - # Copy FROM, WHERE from original - # Use copy() to avoid sharing references between queries - if query.args.get("from_"): - cluster_query.set("from_", query.args["from_"].copy()) - if query.args.get("where"): - cluster_query.set("where", query.args["where"].copy()) - - # Apply CLUSTER transformation to get the CTE-based query - cluster_query = self.cluster_transformer.transform(cluster_query) - - # Build GROUP BY columns - group_by_cols = [exp.column(chrom_col)] - - # Handle stranded parameter - if stranded_expr: - if isinstance(stranded_expr, exp.Boolean): - stranded = stranded_expr.this - elif isinstance(stranded_expr, exp.Literal): - stranded = str(stranded_expr.this).upper() == "TRUE" - else: - stranded = str(stranded_expr).upper() in ("TRUE", "1", "YES") - else: - stranded = False - - if stranded: - group_by_cols.append(exp.column(strand_col, quoted=True)) - - group_by_cols.append(exp.column("__giql_cluster_id")) - - # Build SELECT expressions for merged intervals - select_exprs = [] - - # Add group-by columns (non-aggregated) - select_exprs.append(exp.column(chrom_col, quoted=True)) - if stranded: - select_exprs.append(exp.column(strand_col, quoted=True)) - - # Add merged interval bounds - select_exprs.append( - exp.alias_( - exp.Min(this=exp.column(start_col, quoted=True)), start_col, quoted=False - ) - ) - select_exprs.append( - exp.alias_( - exp.Max(this=exp.column(end_col, quoted=True)), end_col, quoted=False - ) - ) - - # Process other columns from original SELECT - for expression in query.expressions: - # Skip the MERGE expression itself - if isinstance(expression, GIQLMerge): - continue - elif isinstance(expression, exp.Alias) and isinstance( - expression.this, GIQLMerge - ): - continue - # Include other columns (they should be aggregates or in GROUP BY) - else: - select_exprs.append(expression) - - # Build final query - final_query = exp.Select() - final_query.select(*select_exprs, copy=False) - - # FROM the clustered subquery - subquery = exp.Subquery( - this=cluster_query, - alias=exp.TableAlias(this=exp.Identifier(this="clustered")), - ) - final_query.from_(subquery, copy=False) - - # Add GROUP BY - final_query.group_by(*group_by_cols, copy=False) - - # Add ORDER BY (chromosome, start) - final_query.order_by( - exp.Ordered(this=exp.column(chrom_col, quoted=True)), copy=False - ) - final_query.order_by( - exp.Ordered(this=exp.column(start_col, quoted=True)), append=True, copy=False - ) - - return final_query - - class IntersectsBinnedJoinTransformer: """Transform column-to-column INTERSECTS into binned equi-joins. diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 3ff986a..0cd934f 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -22,10 +22,8 @@ from giql.table import Table from giql.table import Tables from giql.targets import resolve_target -from giql.transformer import ClusterTransformer from giql.transformer import IntersectsBinnedJoinTransformer from giql.transformer import IntersectsDuckDBIEJoinTransformer -from giql.transformer import MergeTransformer @overload @@ -210,8 +208,6 @@ def transpile( if duckdb_sql is not None: return duckdb_sql - merge_transformer = MergeTransformer(tables_container) - cluster_transformer = ClusterTransformer(tables_container) generator = BaseGIQLGenerator(tables=tables_container) with _reraise_as_value_error("Transformation error"): @@ -219,19 +215,20 @@ def transpile( # declined the query (returned None) and fell back to the binned plan, # exactly as before. ``intersects_bin_size`` is rejected up front for # iejoin targets, so the binned transformer always sees its default there. + # + # CLUSTER and MERGE used to be rewritten here too; they are now expanded in + # pass 3 (ExpandOperators) by giql.expanders.cluster / .merge (#144). if not target_overrides_intersects: intersects_transformer = IntersectsBinnedJoinTransformer( tables_container, bin_size=intersects_bin_size, ) ast = intersects_transformer.transform(ast) - ast = merge_transformer.transform(ast) - ast = cluster_transformer.transform(ast) # Pass 1 of the normalization pipeline (epic #114): attach resolution - # metadata to every GIQL operator slot ahead of generation. DISJOIN's - # expander consumes this metadata (step 2); the remaining operators still - # use the generator's legacy resolver paths until their ports land. + # metadata to every GIQL operator slot ahead of generation. Every migrated + # operator's expander consumes this metadata in pass 3 (CLUSTER/MERGE carry an + # empty resolution, deriving their columns from the FROM table instead). with _reraise_as_value_error("Resolution error"): ast = resolve_operator_refs(ast, tables_container) diff --git a/tests/expanders/test_cluster.py b/tests/expanders/test_cluster.py new file mode 100644 index 0000000..5fae50a --- /dev/null +++ b/tests/expanders/test_cluster.py @@ -0,0 +1,370 @@ +"""Behavioral tests for the CLUSTER operator expander (#144). + +CLUSTER migrated from the pre-pass ``ClusterTransformer`` to a registered +expander (``giql.expanders.cluster``) that rewrites the enclosing SELECT in place +into the two-level ``lag_calc`` form. These tests drive the public ``transpile`` +API and pin the behaviors the migration newly exercises — the pass walk +replacing the transformer's manual recursion (nested placements), the +FROM-table column derivation, the distance/clause branches, and projection +shapes — that the legacy transpilation suites did not already cover. The +predicate byte-shape is pinned in ``tests/test_cluster_predicate_transpilation.py``. +""" + +import pytest + +from giql.table import Table +from giql.transpile import transpile + + +class TestClusterExpander: + """Expansion of CLUSTER through the operator-expander registry (#144).""" + + @pytest.mark.parametrize( + "query", + [ + "SELECT * FROM (SELECT *, CLUSTER(interval) AS cid FROM peaks) x", + "WITH c AS (SELECT *, CLUSTER(interval) AS cid FROM peaks) SELECT * FROM c", + ], + ) + def test_transpile_should_expand_cluster_nested_in_subquery_or_cte(self, query): + """Test that a CLUSTER nested below the root SELECT still expands. + + Given: + A CLUSTER inside a FROM-subquery, and a CLUSTER inside a WITH CTE. + When: + Transpiling the query. + Then: + The nested CLUSTER should expand into the two-level lag_calc form with + no leaked operator — the pass walk replaces the manual recursion the + legacy transformer performed. + """ + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "G_I_Q_L" not in sql + assert "AS lag_calc" in sql + assert "is_new_cluster" in sql + + def test_transpile_should_raise_when_multiple_cluster_in_one_select(self): + """Test that two CLUSTER expressions in one SELECT are rejected. + + Given: + A SELECT projecting two CLUSTER expressions. + When: + Transpiling the query. + Then: + It should raise ValueError naming the unsupported multiple-CLUSTER + case, rather than chaining the rewrite into non-executable SQL (a + duplicate lag_calc alias / is_new_cluster binder error). + """ + # Arrange + query = ( + "SELECT *, CLUSTER(interval) AS a, CLUSTER(interval, 100) AS b FROM peaks" + ) + + # Act & assert + with pytest.raises( + ValueError, match="Multiple CLUSTER expressions not yet supported" + ): + transpile(query, tables=["peaks"]) + + def test_transpile_should_use_custom_columns_when_table_declares_them(self): + """Test that a stranded CLUSTER honors a custom column mapping. + + Given: + A stranded CLUSTER over a Table declaring custom column names. + When: + Transpiling the query. + Then: + The window partition and order should use the custom column names, not + the canonical defaults. + """ + # Arrange + regions = Table( + "regions", chrom_col="ch", start_col="s", end_col="e", strand_col="st" + ) + query = "SELECT *, CLUSTER(interval, stranded := true) AS cid FROM regions" + + # Act + sql = transpile(query, tables=[regions]) + + # Assert + assert 'PARTITION BY "ch", "st" ORDER BY "s"' in sql + assert 'LAG("e")' in sql + assert '"chrom"' not in sql and '"start"' not in sql + + def test_transpile_should_add_distance_offset_to_lag_when_distance_positive(self): + """Test that a positive CLUSTER distance offsets the adjacency LAG. + + Given: + A CLUSTER with a positive distance. + When: + Transpiling the query. + Then: + The adjacency should add the distance to the LAG before comparing to + start. + """ + # Arrange + query = "SELECT *, CLUSTER(interval, 100) AS cid FROM peaks" + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + window = 'OVER (PARTITION BY "chrom" ORDER BY "start" NULLS LAST)' + assert f'LAG("end") {window} + 100 >= "start"' in sql + + def test_transpile_should_not_offset_lag_when_no_distance(self): + """Test that a CLUSTER without distance uses a bare adjacency. + + Given: + A CLUSTER with no distance argument. + When: + Transpiling the query. + Then: + The adjacency should compare the bare LAG to start with no offset. + """ + # Arrange + query = "SELECT *, CLUSTER(interval) AS cid FROM peaks" + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + window = 'OVER (PARTITION BY "chrom" ORDER BY "start" NULLS LAST)' + assert f'LAG("end") {window} >= "start"' in sql + assert f'{window} + ' not in sql + + def test_transpile_should_split_clauses_between_lag_calc_and_outer_query(self): + """Test that CLUSTER places clauses at the correct query level. + + Given: + A CLUSTER query carrying WHERE, GROUP BY, HAVING, and ORDER BY. + When: + Transpiling the query. + Then: + WHERE/GROUP BY/HAVING should land inside the inner lag_calc subquery + and ORDER BY should attach to the outer query. + """ + # Arrange + query = ( + "SELECT chrom, CLUSTER(interval) AS cid FROM peaks " + "WHERE chrom = 'chr1' GROUP BY chrom HAVING COUNT(*) > 1 ORDER BY chrom" + ) + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert ( + "WHERE chrom = 'chr1' GROUP BY chrom HAVING COUNT(*) > 1) AS lag_calc" in sql + ) + assert ") AS lag_calc ORDER BY chrom" in sql + + def test_transpile_should_expand_bare_cluster_without_alias(self): + """Test that an un-aliased CLUSTER expands to a bare SUM window. + + Given: + A CLUSTER projected without an AS alias. + When: + Transpiling the query. + Then: + The bare CLUSTER should be replaced by an un-aliased SUM window with no + leaked operator. + """ + # Arrange + query = "SELECT *, CLUSTER(interval) FROM peaks" + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "G_I_Q_L" not in sql + assert "SUM(is_new_cluster) OVER" in sql + + def test_transpile_should_keep_explicit_projection_columns_in_lag_calc(self): + """Test that explicit (non-star) projection columns flow into lag_calc. + + Given: + A CLUSTER query with an explicit column projection (not SELECT *). + When: + Transpiling the query. + Then: + The explicit columns should be projected by the inner lag_calc + subquery feeding the cluster window. + """ + # Arrange + query = "SELECT chrom, start, CLUSTER(interval) AS c FROM peaks" + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "SELECT chrom, start," in sql + assert "AS lag_calc" in sql + assert "G_I_Q_L" not in sql + + # Note: CLUSTER combined with an INTERSECTS *join* in the same SELECT is not + # tested here. The clause copy into lag_calc carries only FROM/WHERE/GROUP/ + # HAVING (never JOINs), so a join is dropped — pre-existing legacy behavior + # that #144 preserves byte-for-byte, not a migration concern. + + def test_transpile_should_compose_distance_stranded_and_predicate(self): + """Test that CLUSTER composes distance, stranded, and predicate together. + + Given: + A CLUSTER with a distance, stranded mode, and a PREV-based predicate. + When: + Transpiling the query. + Then: + The output should offset the LAG by the distance, partition by chrom + and strand, and rewrite PREV into a LAG window inside the adjacency. + """ + # Arrange + query = ( + "SELECT *, CLUSTER(interval, 1000, stranded := true, " + "predicate := name = PREV(name)) AS cid FROM peaks" + ) + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "+ 1000 >= " in sql + assert 'PARTITION BY "chrom", "strand"' in sql + assert 'LAG("name")' in sql + assert "G_I_Q_L" not in sql + + def test_transpile_should_expand_cluster_in_union_branch(self): + """Test that a CLUSTER inside a UNION branch is expanded. + + Given: + A CLUSTER in each branch of a UNION (a shape the legacy transformer's + manual recursion did not descend into, leaking an unexpanded operator). + When: + Transpiling the query. + Then: + Both branches should expand to the lag_calc form with no leaked + operator — the pass walk reaches UNION branches. + """ + # Arrange + query = ( + "SELECT *, CLUSTER(interval) AS c FROM peaks " + "UNION ALL SELECT *, CLUSTER(interval) AS c FROM peaks" + ) + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "G_I_Q_L" not in sql + assert sql.count("AS lag_calc") == 2 + + @pytest.mark.parametrize("predicate_op", ["INTERSECTS", "CONTAINS", "WITHIN"]) + @pytest.mark.parametrize( + "projection", + ["*, CLUSTER(interval) AS cid", "CLUSTER(interval)"], + ids=["aliased", "bare"], + ) + def test_transpile_should_expand_spatial_predicate_copied_into_lag_calc( + self, projection, predicate_op + ): + """Test that a spatial WHERE predicate survives the CLUSTER rewrite. + + Given: + A CLUSTER query (aliased or bare) whose WHERE filters on a spatial + predicate, which the rewrite copies into the inner lag_calc subquery. + When: + Transpiling the query. + Then: + The copied predicate should itself be expanded — no leaked, unexpanded + operator — for both projection depths, pinning the #144 B1 regression + where the aliased CLUSTER expanded before the predicate and stranded a + live, unexpanded copy in the subquery. + """ + # Arrange + query = ( + f"SELECT {projection} FROM peaks a " + f"WHERE a.interval {predicate_op} 'chr1:1-100'" + ) + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "G_I_Q_L" not in sql + assert "AS lag_calc" in sql + + @pytest.mark.parametrize( + "query", + [ + "SELECT ABS(CLUSTER(interval)) FROM peaks", + "SELECT CLUSTER(interval) + 1 AS c FROM peaks", + ], + ) + def test_transpile_should_raise_when_cluster_nested_in_projection_expression( + self, query + ): + """Test that a CLUSTER buried in a projection expression is rejected. + + Given: + A CLUSTER nested inside a larger projection expression (a function call + or arithmetic), which has no coherent whole-query rewrite. + When: + Transpiling the query. + Then: + It should raise ValueError requiring a top-level projection item, + rather than leaking an unexpanded operator to the generator. + """ + # Act & assert + with pytest.raises(ValueError, match="must be a top-level projection item"): + transpile(query, tables=["peaks"]) + + def test_transpile_should_inject_lag_calc_columns_deterministically(self): + """Test that the injected lag_calc column order is hash-seed independent. + + Given: + An explicit-projection CLUSTER whose predicate forces several residual + columns to be injected into lag_calc, transpiled in two child + interpreters under differing PYTHONHASHSEED values. + When: + Comparing the two emitted strings. + Then: + They should be byte-identical, proving the injected-column order is + sorted rather than set-iteration (PYTHONHASHSEED) dependent (#144 A2). + """ + # Arrange + import os + import subprocess + import sys + + code = ( + "from giql.transpile import transpile; " + "print(transpile(" + "\"SELECT chrom, CLUSTER(interval, stranded := true, " + "predicate := name = PREV(score)) AS c FROM peaks\", tables=['peaks']))" + ) + base_env = { + k: v + for k, v in os.environ.items() + if not k.startswith("COV_CORE") and k != "COVERAGE_PROCESS_START" + } + + def _run(seed: str) -> str: + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env={**base_env, "PYTHONHASHSEED": seed}, + ) + assert result.returncode == 0, result.stderr + return result.stdout + + # Act + out_a = _run("0") + out_b = _run("1") + + # Assert + assert out_a == out_b + assert "G_I_Q_L" not in out_a diff --git a/tests/expanders/test_merge.py b/tests/expanders/test_merge.py new file mode 100644 index 0000000..fcdc9d7 --- /dev/null +++ b/tests/expanders/test_merge.py @@ -0,0 +1,292 @@ +"""Behavioral tests for the MERGE operator expander (#144). + +MERGE migrated from the pre-pass ``MergeTransformer`` to a registered expander +(``giql.expanders.merge``) that rewrites the enclosing SELECT in place into the +clustered-aggregation form. These tests drive the public ``transpile`` API and +pin the behaviors the migration newly exercises (the registry pass, the +recursion-removal, and the error/limitation guards) that the legacy +transpilation suites did not already cover. The predicate byte-shape is pinned +separately in ``tests/test_cluster_predicate_transpilation.py``. +""" + +import pytest + +from giql.table import Table +from giql.transpile import transpile + + +class TestMergeExpander: + """Expansion of MERGE through the operator-expander registry (#144).""" + + def test_transpile_should_raise_when_multiple_merge_in_one_select(self): + """Test that two MERGE expressions in one SELECT are rejected. + + Given: + A SELECT projecting two MERGE expressions. + When: + Transpiling the query. + Then: + It should raise ValueError naming the unsupported multiple-MERGE case. + """ + # Arrange + query = "SELECT MERGE(interval), MERGE(interval, 100) FROM peaks" + + # Act & assert + with pytest.raises( + ValueError, match="Multiple MERGE expressions not yet supported" + ): + transpile(query, tables=["peaks"]) + + @pytest.mark.parametrize( + "query", + [ + "SELECT MERGE(interval), CLUSTER(interval) AS cid FROM peaks", + "SELECT CLUSTER(interval) AS cid, MERGE(interval) FROM peaks", + ], + ) + def test_transpile_should_raise_when_cluster_and_merge_share_select(self, query): + """Test that combining CLUSTER and MERGE in one SELECT is rejected. + + Given: + A SELECT projecting both a MERGE and a CLUSTER (either order). + When: + Transpiling the query. + Then: + It should raise ValueError naming the unsupported combination, rather + than silently emitting SQL with a leaked, unexpanded operator. + """ + # Act & assert + with pytest.raises(ValueError, match="CLUSTER and MERGE cannot be combined"): + transpile(query, tables=["peaks"]) + + def test_transpile_should_group_by_strand_when_merge_stranded(self): + """Test that a stranded MERGE aggregates within strand. + + Given: + A stranded MERGE query. + When: + Transpiling the query. + Then: + It should aggregate MIN(start)/MAX(end), group by chrom, strand, and + the synthesized cluster id, project strand, and partition the inner + windows by chrom and strand. + """ + # Arrange + query = "SELECT MERGE(interval, stranded := true) FROM peaks" + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert 'MIN("start") AS start' in sql + assert 'MAX("end") AS end' in sql + assert 'GROUP BY "chrom", "strand", __giql_cluster_id' in sql + assert 'PARTITION BY "chrom", "strand"' in sql + assert "G_I_Q_L" not in sql + + def test_transpile_should_use_custom_columns_when_table_declares_them(self): + """Test that a stranded MERGE honors a custom column mapping. + + Given: + A stranded MERGE over a Table declaring custom chrom/start/end/strand + column names. + When: + Transpiling the query. + Then: + The aggregation, GROUP BY, and window partitions should use the custom + column names, never the canonical defaults. + """ + # Arrange + regions = Table( + "regions", chrom_col="ch", start_col="s", end_col="e", strand_col="st" + ) + query = "SELECT MERGE(interval, stranded := true) FROM regions" + + # Act + sql = transpile(query, tables=[regions]) + + # Assert + assert 'MIN("s") AS s' in sql + assert 'MAX("e") AS e' in sql + assert 'GROUP BY "ch", "st", __giql_cluster_id' in sql + assert 'PARTITION BY "ch", "st"' in sql + assert '"chrom"' not in sql and '"start"' not in sql + + @pytest.mark.parametrize( + "query", + [ + "SELECT * FROM (SELECT MERGE(interval) FROM peaks) x", + "WITH c AS (SELECT MERGE(interval) FROM peaks) SELECT * FROM c", + ], + ) + def test_transpile_should_expand_merge_nested_in_subquery_or_cte(self, query): + """Test that a MERGE nested below the root SELECT still expands. + + Given: + A MERGE inside a FROM-subquery, and a MERGE inside a WITH CTE. + When: + Transpiling the query. + Then: + The nested MERGE should expand into the clustered-aggregation form + with no leaked operator — the pass walk replaces the manual recursion + the legacy transformer performed. + """ + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "G_I_Q_L" not in sql + assert "AS clustered" in sql + assert "__giql_cluster_id" in sql + + def test_transpile_should_carry_where_into_clustered_subquery(self): + """Test that a WHERE clause is pushed into MERGE's clustered subquery. + + Given: + A MERGE query with a WHERE clause. + When: + Transpiling the query. + Then: + The WHERE predicate should appear inside the inner clustered subquery + that feeds the aggregation. + """ + # Arrange + query = "SELECT MERGE(interval) AS m FROM peaks WHERE chrom = 'chr1'" + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "WHERE chrom = 'chr1'" in sql + assert "AS clustered" in sql + + @pytest.mark.parametrize( + "predicate, message", + [ + ("depth = PREV(depth, score)", "exactly one column argument"), + ("depth = PREV(PREV(depth))", "cannot be nested"), + ], + ) + def test_transpile_should_validate_prev_in_merge_predicate(self, predicate, message): + """Test that MERGE reuses CLUSTER's PREV validation. + + Given: + A MERGE predicate calling PREV with the wrong arity or a nested PREV. + When: + Transpiling the query. + Then: + It should raise the matching ValueError, proving the MERGE path + reaches the shared predecessor-reference validation. + """ + # Arrange + query = f"SELECT MERGE(interval, predicate := {predicate}) FROM peaks" + + # Act & assert + with pytest.raises(ValueError, match=message): + transpile(query, tables=["peaks"]) + + def test_transpile_should_expand_merge_in_projection_scalar_subquery(self): + """Test that a MERGE in a projection scalar-subquery is expanded. + + Given: + A MERGE inside a scalar subquery in the SELECT list (a shape the legacy + transformer's manual recursion did not descend into, leaking an + unexpanded operator). + When: + Transpiling the query. + Then: + The MERGE should expand into the clustered-aggregation form with no + leaked operator — the pass walk reaches projection subqueries. + """ + # Arrange + query = "SELECT (SELECT MERGE(interval) FROM peaks) AS m FROM other" + + # Act + sql = transpile(query, tables=["peaks", "other"]) + + # Assert + assert "G_I_Q_L" not in sql + assert "AS clustered" in sql + assert "__giql_cluster_id" in sql + + @pytest.mark.parametrize("predicate_op", ["INTERSECTS", "CONTAINS", "WITHIN"]) + @pytest.mark.parametrize( + "projection", + ["*, MERGE(interval) AS m", "MERGE(interval)"], + ids=["aliased", "bare"], + ) + def test_transpile_should_expand_spatial_predicate_copied_into_clustered( + self, projection, predicate_op + ): + """Test that a spatial WHERE predicate survives the MERGE rewrite. + + Given: + A MERGE query (aliased or bare) whose WHERE filters on a spatial + predicate, which the rewrite copies into the inner clustered subquery. + When: + Transpiling the query. + Then: + The copied predicate should itself be expanded — no leaked, unexpanded + operator — for both projection depths, pinning the #144 B1 regression + where the aliased MERGE expanded before the predicate and stranded a + live, unexpanded copy in the subquery. + """ + # Arrange + query = ( + f"SELECT {projection} FROM peaks a " + f"WHERE a.interval {predicate_op} 'chr1:1-100'" + ) + + # Act + sql = transpile(query, tables=["peaks"]) + + # Assert + assert "G_I_Q_L" not in sql + assert "AS clustered" in sql + + @pytest.mark.parametrize( + "query", + [ + "SELECT ABS(MERGE(interval)) FROM peaks", + "SELECT MERGE(interval) + 1 AS m FROM peaks", + ], + ) + def test_transpile_should_raise_when_merge_nested_in_projection_expression( + self, query + ): + """Test that a MERGE buried in a projection expression is rejected. + + Given: + A MERGE nested inside a larger projection expression (a function call or + arithmetic), which has no coherent whole-query rewrite. + When: + Transpiling the query. + Then: + It should raise ValueError requiring a top-level projection item, + rather than leaking an unexpanded operator to the generator. + """ + # Act & assert + with pytest.raises(ValueError, match="must be a top-level projection item"): + transpile(query, tables=["peaks"]) + + def test_transpile_should_quote_group_by_chrom_when_chrom_is_reserved_word(self): + """Test that the MERGE GROUP BY chrom term is quoted. + + Given: + A MERGE over a Table whose chrom column is a SQL reserved word. + When: + Transpiling the query. + Then: + The GROUP BY chrom term should be quoted (like every other chrom + reference), so the reserved-word column emits valid SQL (#144 A13). + """ + # Arrange + regions = Table("regions", chrom_col="order", start_col="s", end_col="e") + query = "SELECT MERGE(interval) FROM regions" + + # Act + sql = transpile(query, tables=[regions]) + + # Assert + assert 'GROUP BY "order"' in sql + assert "GROUP BY order" not in sql diff --git a/tests/test_expander.py b/tests/test_expander.py index ab93732..7844552 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -1172,44 +1172,38 @@ def test_expand_operators_is_identity_when_registry_empty(self): assert result is ast assert list(result.find_all(GIQLDisjoin)) - def test_transpile_sql_unchanged_with_pass_inert(self): - """Test that transpile output is byte-identical for an unmigrated operator. + def test_expand_operators_skips_opted_out_operator_with_default_registry(self): + """Test that the pass skips an opted-out operator even with its expander live. Given: - A CLUSTER query (an operator not migrated onto the pass in any wave-3 - branch, so its GIQL_EXPAND is False and no expander resolves), with - the default registry. + A migrated operator query and the import-populated default REGISTRY (so + its expander resolves), but the operator opted out of GIQL_EXPAND for + the test. When: - Running the ExpandOperators pass (default REGISTRY) over the resolved - AST and serializing both the original and the pass-run AST. + Running the ExpandOperators pass over the default REGISTRY. Then: - The pass leaves the operator node in place, the serialized SQL is - byte-identical, and no expander alias prefix appears — the pass is - inert for any operator that has not been migrated. + It should leave the operator node in place and return the same tree — + the per-type GIQL_EXPAND gate holds even when an expander is registered + (after #144 no operator ships unmigrated, so the gate is exercised via + an opt-out rather than a shipped False). """ # Arrange - query = "SELECT *, CLUSTER(interval) AS cluster_id FROM peaks" - tables = _tables(("peaks",)) - ast = _prepare(query, tables) - from giql.generators import BaseGIQLGenerator - - before = BaseGIQLGenerator(tables=tables).generate(ast) - before_ops = len(list(ast.find_all(GIQLCluster))) + operator = _A_MIGRATED_OPERATOR + ast, tables = _prepare_operator(operator) + before_ops = len(list(ast.find_all(operator))) # Act — the wired-in pass over the default REGISTRY must be a no-op here. - result = expand_operators(ast, GenericTarget(), tables) - after = BaseGIQLGenerator(tables=tables).generate(result) + with _opted_out(operator): + result = expand_operators(ast, GenericTarget(), tables) # Assert - assert after == before - assert len(list(result.find_all(GIQLCluster))) == before_ops - assert EXPAND_ALIAS_PREFIX not in after + assert result is ast + assert len(list(result.find_all(operator))) == before_ops # The nine GIQL operator expression classes the ExpandOperators pass inspects. -# Each migrated operator ships opted in (GIQL_EXPAND=True) alongside its -# registered expander; the rest ship opted out (False) and fall through to the -# legacy emitter. +# Every operator is now migrated (#144): each ships opted in (GIQL_EXPAND=True) +# alongside its registered expander, so none falls through to a legacy emitter. from giql.expressions import Contains # noqa: E402 from giql.expressions import GIQLCluster # noqa: E402 from giql.expressions import GIQLDistance # noqa: E402 @@ -1252,11 +1246,15 @@ def test_transpile_sql_unchanged_with_pass_inert(self): assert _MIGRATED_OPERATORS, "expected at least one migrated operator" #: An arbitrary migrated operator the operator-agnostic control tests target. _A_MIGRATED_OPERATOR = _MIGRATED_OPERATORS[0] -#: Operators not yet migrated — they ship GIQL_EXPAND=False. +#: Operators not yet migrated — they ship GIQL_EXPAND=False. Empty since #144 +#: migrated the last two (CLUSTER and MERGE): every operator now expands through +#: the pass. Control tests that need an operator to behave as if unmigrated drive a +#: migrated one through ``_opted_out`` rather than relying on a shipped ``False``. _UNMIGRATED_OPERATORS = tuple( op for op in _OPERATOR_CLASSES if op not in _MIGRATED_OPERATORS ) -assert _UNMIGRATED_OPERATORS, "expected at least one unmigrated operator" +#: Pin the post-#144 invariant: every GIQL operator is migrated onto the pass. +assert not _UNMIGRATED_OPERATORS, "every operator should be migrated after #144" #: A minimal GIQL query producing one node of each operator class, keyed by the @@ -1314,28 +1312,90 @@ def _prepare_operator(operator: type) -> tuple[exp.Expression, Tables]: return _prepare(query, tables), tables -class TestOperatorOptOut: - """Migrated operators opt into the pass; the rest still ship opted out.""" +class TestClusterMergeExpansion: + """CLUSTER and MERGE expand through the pass into their restructured forms (#144).""" - @pytest.mark.parametrize( - "operator", _UNMIGRATED_OPERATORS, ids=lambda c: c.__name__ - ) - def test_operator_class_ships_expand_disabled(self, operator): - """Test that each unmigrated operator class ships GIQL_EXPAND=False. + def test_transform_replaces_cluster_with_lag_calc_subquery(self): + """Test that the pass rewrites a CLUSTER query into the two-level form. Given: - A GIQL operator expression class that has not been migrated onto the - ExpandOperators pass. + A resolved ``SELECT *, CLUSTER(interval) ...`` AST and the default + REGISTRY (CLUSTER ships GIQL_EXPAND=True with a registered expander). When: - Reading its GIQL_EXPAND class attribute. + Running the ExpandOperators pass. Then: - It should be False (the operator still uses the legacy emitter). + It should consume the CLUSTER node in place (returning the same root + object), wrap the source in a ``lag_calc`` derived table with a LAG + window and an ``is_new_cluster`` CASE, project an outer SUM window, and + mint no expander alias. """ - # Arrange & act - flag = operator.GIQL_EXPAND + # Arrange + ast, tables = _prepare_operator(GIQLCluster) + + # Act + result = expand_operators(ast, GenericTarget(), tables) # Assert - assert flag is False + assert result is ast # whole-query rewrite mutates the root in place + assert not list(result.find_all(GIQLCluster)) + aliases = {sub.alias for sub in result.find_all(exp.Subquery) if sub.alias} + assert "lag_calc" in aliases + windows = list(result.find_all(exp.Window)) + assert any(isinstance(w.this, exp.Sum) for w in windows) # outer cluster id + assert any( + isinstance(w.this, exp.Anonymous) and w.this.name.upper() == "LAG" + for w in windows + ) # inner adjacency LAG + assert any( + isinstance(a, exp.Alias) and a.alias == "is_new_cluster" + for a in result.find_all(exp.Alias) + ) + assert EXPAND_ALIAS_PREFIX not in result.sql(dialect=GIQLDialect) + + def test_transform_replaces_merge_with_clustered_group_by(self): + """Test that the pass rewrites a MERGE query into the clustered-aggregation form. + + Given: + A resolved ``SELECT MERGE(interval) ...`` AST and the default REGISTRY + (MERGE ships GIQL_EXPAND=True with a registered expander). + When: + Running the ExpandOperators pass. + Then: + It should consume the MERGE node in place (returning the same root + object), wrap a ``clustered`` subquery (itself wrapping a ``lag_calc``) + under a GROUP BY that includes the synthesized ``__giql_cluster_id``, + project MIN/MAX bounds, and mint no expander alias. + """ + # Arrange + ast, tables = _prepare_operator(GIQLMerge) + + # Act + result = expand_operators(ast, GenericTarget(), tables) + + # Assert + assert result is ast # whole-query rewrite mutates the root in place + assert not list(result.find_all(GIQLMerge)) + aliases = {sub.alias for sub in result.find_all(exp.Subquery) if sub.alias} + assert "clustered" in aliases # MERGE wraps the clustered subquery + assert "lag_calc" in aliases # built on CLUSTER + group = result.find(exp.Group) + assert group is not None + assert any( + isinstance(g, exp.Column) and g.name == "__giql_cluster_id" + for g in group.expressions + ) + assert any(isinstance(m, exp.Min) for m in result.find_all(exp.Min)) + assert any(isinstance(m, exp.Max) for m in result.find_all(exp.Max)) + assert EXPAND_ALIAS_PREFIX not in result.sql(dialect=GIQLDialect) + + +class TestOperatorOptOut: + """Every operator is now migrated, so all ship GIQL_EXPAND=True. + + The complementary ``test_operator_class_ships_expand_disabled`` was dropped + when #144 migrated the last operators: ``_UNMIGRATED_OPERATORS`` is empty, so + there is no class left to assert ships ``False``. + """ @pytest.mark.parametrize( "operator", _MIGRATED_OPERATORS, ids=lambda c: c.__name__ @@ -1392,24 +1452,27 @@ def test_opted_in_restores_flag_after_exception(self): """Test that _opted_in restores GIQL_EXPAND when the body raises. Given: - An operator class at its default GIQL_EXPAND=False. + An operator driven to GIQL_EXPAND=False via _opted_out (every operator + now ships True after #144, so the restore target is set up explicitly). When: Its _opted_in body raises an exception. Then: The flag should be restored to False (the manager is exception-safe, so a raising expansion test cannot leak an opt-in into a later test). """ - # Arrange - assert GIQLMerge.GIQL_EXPAND is False + # Arrange — set the restore target to False so the restore is observable. + operator = _A_MIGRATED_OPERATOR + with _opted_out(operator): + assert operator.GIQL_EXPAND is False - # Act - with pytest.raises(RuntimeError): - with _opted_in(GIQLMerge): - assert GIQLMerge.GIQL_EXPAND is True - raise RuntimeError("boom") + # Act + with pytest.raises(RuntimeError): + with _opted_in(operator): + assert operator.GIQL_EXPAND is True + raise RuntimeError("boom") - # Assert - assert GIQLMerge.GIQL_EXPAND is False + # Assert + assert operator.GIQL_EXPAND is False class TestIEJoinRegistryDeferral: @@ -1938,22 +2001,19 @@ def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): """Test that only the flagged operator type is replaced when both registered. Given: - A genuinely-unmigrated operator (GIQLCluster, shipping - GIQL_EXPAND=False in every wave-3 branch) and an INTERSECTS, both with - registered expanders, but only INTERSECTS flagged GIQL_EXPAND for the - test. + A CLUSTER and an INTERSECTS, both with registered expanders, but + CLUSTER held off (opted out of GIQL_EXPAND) while only INTERSECTS is + opted in. When: Running the pass. Then: - The INTERSECTS is replaced while the unmigrated operator node remains - on its own shipped ``False`` flag — no opt-out ceremony needed (the - gate is per-type). + The INTERSECTS is replaced while the held-off operator node remains — + the gate is per-type, so opting CLUSTER out alone holds its expansion + off even though its expander is registered. """ - # Arrange — the held-off subject is genuinely unmigrated: it survives on - # its own shipped GIQL_EXPAND=False, not on a test opt-out. + # Arrange — the held-off subject is a migrated operator driven off via + # _opted_out (after #144 no operator ships GIQL_EXPAND=False). held_off = GIQLCluster - assert held_off in _UNMIGRATED_OPERATORS - assert held_off.GIQL_EXPAND is False clean_registry.register( GenericTarget(), held_off, lambda n, c: exp.column("CL") ) @@ -1968,9 +2028,10 @@ def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): ) pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - # Act — only INTERSECTS is opted in; the unmigrated operator stays off. - with _opted_in(Intersects): - result = pass_.transform(ast) + # Act — only INTERSECTS is opted in; CLUSTER is held off via opt-out. + with _opted_out(held_off): + with _opted_in(Intersects): + result = pass_.transform(ast) # Assert assert list(result.find_all(held_off)) diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 5bf6138..754474e 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -238,6 +238,7 @@ def test_cluster_basic(self): assert "SELECT" in sql assert "SUM" in sql.upper() or "LAG" in sql.upper() + assert "G_I_Q_L" not in sql # no leaked, unexpanded operator def test_cluster_with_distance(self): """ @@ -255,6 +256,7 @@ def test_cluster_with_distance(self): assert "SELECT" in sql assert "100" in sql + assert "G_I_Q_L" not in sql # no leaked, unexpanded operator def test_cluster_stranded(self): """ @@ -272,6 +274,7 @@ def test_cluster_stranded(self): assert "SELECT" in sql assert "strand" in sql.lower() + assert "G_I_Q_L" not in sql # no leaked, unexpanded operator class TestTranspileMerge: @@ -292,6 +295,7 @@ def test_merge_basic(self): assert "MIN" in sql.upper() assert "MAX" in sql.upper() assert "GROUP BY" in sql.upper() + assert "G_I_Q_L" not in sql # no leaked, unexpanded operator def test_merge_with_distance(self): """ @@ -306,6 +310,7 @@ def test_merge_with_distance(self): assert "SELECT" in sql assert "100" in sql + assert "G_I_Q_L" not in sql # no leaked, unexpanded operator def test_merge_with_aggregation(self): """ @@ -320,6 +325,7 @@ def test_merge_with_aggregation(self): assert "SELECT" in sql assert "COUNT" in sql.upper() + assert "G_I_Q_L" not in sql # no leaked, unexpanded operator class TestTranspileNearest: From 07d6cade71f71ca5ac7ce2af18bc48dfc5ef4f1f Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 30 Jun 2026 12:36:39 -0400 Subject: [PATCH 098/142] refactor: Make coordinate canonicalization capability-driven MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the DataFusion target (epic #137) by choosing the coordinate canonicalization emit strategy from the active target's capabilities rather than hardcoding a DuckDB-only star REPLACE. The CanonicalizeCoordinates pass now takes the target's Capabilities, and both the wrapper-CTE projection and the NEAREST row passthrough emit a star REPLACE when supports_star_replace holds (DuckDB) and the portable star EXCEPT form otherwise (the generic / DataFusion family). This removes the two remaining hardcoded REPLACE assumptions — the canonicalizer wrapper and the NEAREST passthrough — so a non-canonical coordinate encoding transpiles to engine-runnable SQL on DataFusion. The capability is threaded from transpile through the pass and through the NEAREST expander; a direct caller that passes no capabilities keeps the REPLACE form, preserving historical behavior. DuckDB output is byte-unchanged. The generic and DataFusion targets now emit the portable EXCEPT form, which is row-equivalent but not column-order-equivalent (EXCEPT re-appends the recomputed interval columns). This is verified end-to-end on the real DataFusion engine across all four coordinate encodings, custom interval-column names, and strand. DataFusion serialization is finalized: it uses the generic sqlglot output, validated by the cross-target oracle, and its capability values are promoted from provisional to verified. The SELECT * over a correlated NEAREST column leak remains deferred to a later query-level seam. --- docs/transpilation/schema-mapping.rst | 36 +-- src/giql/canonicalizer.py | 128 +++++---- src/giql/expanders/nearest.py | 20 +- src/giql/generators/base.py | 42 ++- src/giql/targets.py | 58 ++-- src/giql/transpile.py | 22 +- tests/generators/test_base.py | 70 +++-- .../datafusion/test_cross_target_encodings.py | 252 ++++++++++++++++++ tests/test_canonicalizer.py | 181 ++++++++++++- tests/test_disjoin_transpilation.py | 13 +- 10 files changed, 682 insertions(+), 140 deletions(-) create mode 100644 tests/integration/datafusion/test_cross_target_encodings.py diff --git a/docs/transpilation/schema-mapping.rst b/docs/transpilation/schema-mapping.rst index 3dbcdef..b991e45 100644 --- a/docs/transpilation/schema-mapping.rst +++ b/docs/transpilation/schema-mapping.rst @@ -175,24 +175,28 @@ If your data uses 1-based coordinates (like VCF or GFF), configure the .. note:: - **Non-canonical encodings currently require a DuckDB-compatible engine.** + **Non-canonical encodings emit a capability-driven canonicalization wrapper.** When a table declares an encoding other than the default 0-based half-open (for example ``coordinate_system="1based"`` or ``interval_type="closed"``), - GIQL canonicalizes its coordinates by wrapping the relation in a hidden CTE - that uses ``SELECT * REPLACE (...)`` syntax. That syntax is supported by - DuckDB, BigQuery, Snowflake, and ClickHouse, but **not** by PostgreSQL, - SQLite, or DataFusion. Tables in the default 0-based half-open encoding are - unaffected -- they take an identity fast path that emits portable SQL. - - To target a non-``REPLACE`` engine today, store your data in 0-based - half-open form, or convert it explicitly in a CTE and reference that CTE - (which GIQL treats as already canonical). Such a CTE -- and any CTE or - subquery passed as an operator reference -- must project the canonical - ``chrom`` / ``start`` / ``end`` columns; GIQL validates this contract at - transpile time and raises a ``ValueError`` naming the missing column(s) - rather than emitting SQL that fails with an engine ``column not found`` - error. Making canonicalization emit portable SQL on every engine is tracked - in `#132 `_. + GIQL canonicalizes its coordinates by wrapping the relation in a hidden CTE. + The wrapper's projection form is chosen from the target's capabilities: the + ``"duckdb"`` target emits ``SELECT * REPLACE (...)`` (also supported by + BigQuery, Snowflake, and ClickHouse), while the generic (``dialect=None``) + and ``"datafusion"`` targets emit the portable ``SELECT * EXCEPT (start, end), + , `` form. The ``* EXCEPT`` form runs on ``* EXCEPT``-capable + engines (the DataFusion family) but is **not** SQL-92 and is **not** + DuckDB-runnable; it is row-equivalent to the ``* REPLACE`` form but re-appends + the recomputed interval columns at the end of the projection. Tables in the + default 0-based half-open encoding are unaffected -- they take an identity fast + path that emits portable SQL on every target. + + Neither form is SQL-92. To target a strict SQL-92 engine (PostgreSQL, SQLite), + store your data in 0-based half-open form, or convert it explicitly in a CTE + and reference that CTE (which GIQL treats as already canonical). Such a CTE -- + and any CTE or subquery passed as an operator reference -- must project the + canonical ``chrom`` / ``start`` / ``end`` columns; GIQL validates this contract + at transpile time and raises a ``ValueError`` naming the missing column(s) + rather than emitting SQL that fails with an engine ``column not found`` error. Working with Point Features ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/giql/canonicalizer.py b/src/giql/canonicalizer.py index 78e1948..f3e3061 100644 --- a/src/giql/canonicalizer.py +++ b/src/giql/canonicalizer.py @@ -34,31 +34,44 @@ tradeoff the epic calls out (only synthesize a wrapper when canonicalization actually changes columns). -Engine portability (known limitation) -------------------------------------- -The wrapper projection uses ``SELECT * REPLACE (...)`` to canonicalize the -interval columns in place while passing every other source column through -untouched (the registry declares only the genomic columns, so an explicit -full-column projection is not available). ``* REPLACE`` is supported by DuckDB, -BigQuery, Snowflake, and ClickHouse, but **not** by PostgreSQL, SQLite, or -DataFusion — so a non-canonical encoding currently transpiles to -engine-incompatible SQL on those targets. Identity-encoded (default 0-based -half-open) relations are unaffected: they skip wrapping entirely and emit -portable SQL. Making the emit strategy dialect-aware (an explicit portable -projection when the target lacks ``REPLACE`` or the full schema is declared) is -tracked in https://github.com/abdenlab/giql/issues/132. +Engine portability (capability-driven, issue #145) +--------------------------------------------------- +The wrapper projection canonicalizes the interval columns while passing every +other source column through untouched (the registry declares only the genomic +columns, so an explicit full-column projection is not available). The emit +strategy is chosen from the active target's :class:`~giql.targets.Capabilities`, +following the DISJOIN passthrough's precedent (#143) — the same two emit forms, +though this wrapper additionally accepts ``capabilities=None`` (a direct caller +default, see :func:`canonicalize_coordinates`): + +* ``SELECT * REPLACE (...)`` when ``capabilities.supports_star_replace`` holds + (DuckDB / BigQuery / Snowflake / ClickHouse) — substitutes start/end in place, + preserving source column order; +* the portable ``SELECT * EXCEPT (start, end), , `` form otherwise + (the generic baseline / DataFusion family), which every ``* EXCEPT``-capable + engine plans. This form is row-equivalent but **not column-order-equivalent**: + ``* EXCEPT`` drops the interval columns and re-appends the recomputed ones at + the end of the projection. It is also not SQL-92 and not DuckDB-runnable. + +Identity-encoded (default 0-based half-open) relations are unaffected either way: +they skip wrapping entirely and emit portable SQL. The capability is threaded in +from :func:`giql.transpile.transpile` via the active target; a direct caller that +passes no capabilities defaults to the ``* REPLACE`` form (the historical +behavior). This finalizes the dialect-aware emit strategy formerly tracked by +https://github.com/abdenlab/giql/issues/132. Gating (epic #114, step 6) -------------------------- The pass is gated per operator by a ``GIQL_CANONICALIZE`` class attribute on the operator's expression class. An operator opts in by setting -``GIQL_CANONICALIZE = True``; absent or ``False`` (the default for every operator -as of this issue) the pass ignores it entirely. The operator port issues — #122 -(DISJOIN) and #123 (NEAREST / DISTANCE / predicates) — flip these flags as each -operator's emitter is moved off in-emitter canonicalization -(:mod:`giql.canonical`) and onto this pass's output. **With every flag off the -pass is a strict no-op and the emitted SQL is byte-identical**, so the existing -suite is the migration oracle. +``GIQL_CANONICALIZE = True``; absent or ``False`` the pass ignores it entirely. +The operator port issues — #122 (DISJOIN) and #123 (NEAREST / DISTANCE / +predicates) — flipped these flags as each operator's emitter moved off in-emitter +canonicalization (:mod:`giql.canonical`) and onto this pass's output. As of those +ports every migrated operator opts in by default, so the pass actively +synthesizes wrappers; an operator can still toggle its flag off (a test or a +not-yet-migrated operator), in which case the pass leaves it untouched and the +emitted SQL is byte-identical for it. De-canonicalization hook ------------------------- @@ -124,6 +137,7 @@ from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef from giql.table import Table +from giql.targets import Capabilities __all__ = [ "CANON_PREFIX", @@ -149,7 +163,9 @@ ) -def canonicalize_coordinates(expression: exp.Expression) -> exp.Expression: +def canonicalize_coordinates( + expression: exp.Expression, capabilities: Capabilities | None = None +) -> exp.Expression: """Synthesize canonical wrapper CTEs for non-canonical operator operands. Walks *expression* for opted-in GIQL operators (those whose expression class @@ -159,20 +175,28 @@ def canonicalize_coordinates(expression: exp.Expression) -> exp.Expression: half-open coordinates and rewrites the slot (AST node + ``ResolvedRef`` metadata) to point at the canonical CTE. - The pass mutates and returns *expression* in place. **When no operator opts - in — the state as of issue #121 — it is a strict no-op: no node is touched - and the emitted SQL is byte-identical.** + The pass mutates and returns *expression* in place. For an operator whose + ``GIQL_CANONICALIZE`` flag is off, or whose operands are already in the + canonical 0-based half-open encoding, it touches nothing and leaves the + emitted SQL byte-identical. Parameters ---------- expression : exp.Expression The pass-1-annotated AST. + capabilities : Capabilities | None + The active target's capabilities, used to choose the wrapper projection's + emit strategy (``* REPLACE`` vs the portable ``* EXCEPT`` form — see the + module docstring). :func:`giql.transpile.transpile` passes the active + target's capabilities; ``None`` (a direct caller) defaults to the + ``* REPLACE`` form, preserving the historical behavior. Returns ------- exp.Expression - The same *expression*, with canonical wrapper CTEs inserted and migrated - operator slots rewritten (none, while every flag is off). + The same *expression*, with canonical wrapper CTEs inserted and the + opted-in operator slots that reference non-canonical tables rewritten to + point at them. """ # Column / interval operands (DISTANCE, predicates, NEAREST's non-table # reference) canonicalize their metadata in place; this is independent of the @@ -192,7 +216,7 @@ def canonicalize_coordinates(expression: exp.Expression) -> exp.Expression: new_ctes: list[exp.CTE] = [] for node, arg, ref in targets: - body = _canonical_projection(ref) + body = _canonical_projection(ref, capabilities) body_sql = body.sql() name = body_to_name.get(body_sql) if name is None: @@ -394,15 +418,28 @@ def _fresh_name(next_name, taken: set[str]) -> str: return candidate -def _canonical_projection(ref: ResolvedRef) -> exp.Select: +def _canonical_projection( + ref: ResolvedRef, capabilities: Capabilities | None +) -> exp.Select: """Build the ``SELECT`` body that projects *ref*'s table to canonical form. The projection is a **full-row passthrough**: ``SELECT *`` keeps every - physical column of the source relation, and a star ``REPLACE`` rewrites only - the two interval columns — ``start`` / ``end``, under their original physical - names — with the :mod:`giql.canonical` arithmetic for the table's declared - encoding. ``chrom`` and every non-interval column flow through the star - untouched. + physical column of the source relation, and only the two interval columns — + ``start`` / ``end``, under their original physical names — are rewritten with + the :mod:`giql.canonical` arithmetic for the table's declared encoding. + ``chrom`` and every non-interval column flow through the star untouched. + + The emit strategy is chosen from *capabilities* (issue #145), following the + precedent of :func:`giql.expanders.disjoin._disjoin_passthrough` — the same + two emit forms, with an added ``capabilities is None`` arm for direct callers + (the passthrough always receives a concrete ``ctx.capabilities``): + + * ``SELECT * REPLACE (...)`` when ``supports_star_replace`` holds (or no + capabilities are supplied) — substitutes the interval columns in place, + preserving source column order; + * the portable ``SELECT * EXCEPT (start, end), , `` form otherwise + — drops the interval columns from the star and re-appends them recomputed. + Row-equivalent but not column-order-equivalent, and not DuckDB-runnable. The full row (rather than a bare ``chrom`` / ``start`` / ``end`` triple) is required by table-function operators whose final projection passes the whole @@ -417,19 +454,20 @@ def _canonical_projection(ref: ResolvedRef) -> exp.Select: # Quote the interval identifiers: the canonical column names are physical and # routinely reserved words (the default genomic layout's ``start`` / ``end``), # so the executed wrapper must quote them. - star = exp.Star( - replace=[ - exp.alias_( - _canonical_start_expr(start, table), - exp.to_identifier(start, quoted=True), - ), - exp.alias_( - _canonical_end_expr(end, table), - exp.to_identifier(end, quoted=True), - ), - ] + start_id = exp.to_identifier(start, quoted=True) + end_id = exp.to_identifier(end, quoted=True) + start_proj = exp.alias_(_canonical_start_expr(start, table), start_id) + end_proj = exp.alias_(_canonical_end_expr(end, table), end_id) + if capabilities is None or capabilities.supports_star_replace: + star = exp.Star(replace=[start_proj, end_proj]) + return exp.Select(expressions=[star]).from_(exp.to_table(relation)) + # Portable form: drop the interval columns from the star and re-project them + # recomputed under their own names. EXCEPT removes them from the row; the + # trailing projections add them back in canonical form. + star = exp.Star(except_=[exp.column(start_id), exp.column(end_id)]) + return exp.Select(expressions=[star, start_proj, end_proj]).from_( + exp.to_table(relation) ) - return exp.Select(expressions=[star]).from_(exp.to_table(relation)) def _canonical_start_expr(start: str, table: Table | None) -> exp.Expression: diff --git a/src/giql/expanders/nearest.py b/src/giql/expanders/nearest.py index 482c232..c2144e8 100644 --- a/src/giql/expanders/nearest.py +++ b/src/giql/expanders/nearest.py @@ -44,6 +44,7 @@ from giql.generators.base import BaseGIQLGenerator from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef +from giql.targets import Capabilities from giql.targets import GenericTarget #: Reserved column names the window-function fallback synthesizes inside its @@ -75,6 +76,7 @@ def _distance_and_filters( table_name: str, target_ref: ResolvedRef, ref: ResolvedInterval, + capabilities: Capabilities, ref_fragments: tuple[str, str, str, str | None] | None = None, ) -> tuple[str, str, list[str], str]: """Build the shared distance SQL, the qualified target columns, and WHERE. @@ -86,6 +88,11 @@ def _distance_and_filters( ``giqlnearest_sql`` emitter exactly. Each form derives its deterministic ORDER BY tiebreaker from the target columns itself. + ``capabilities`` is the active target's :class:`~giql.targets.Capabilities`, + forwarded to :meth:`BaseGIQLGenerator._nearest_passthrough` to choose the + target's de-canonicalization emit form (``* REPLACE`` vs the portable + ``* EXCEPT``); both call sites pass ``ctx.capabilities``. + ``ref_fragments`` optionally overrides the reference ``(chrom, start, end, strand)`` SQL fragments. The LATERAL form consumes the resolution's outer-qualified fragments verbatim; the fallback passes fragments pointing at @@ -98,7 +105,7 @@ def _distance_and_filters( output_table = BaseGIQLGenerator._nearest_output_encoding(expression, target_ref) passthrough = BaseGIQLGenerator._nearest_passthrough( - table_name, target_start, target_end, output_table + table_name, target_start, target_end, output_table, capabilities ) if ref_fragments is not None: @@ -189,7 +196,9 @@ def _lateral_form( _abs_distance_expr, where_clauses, passthrough, - ) = _distance_and_filters(expression, table_name, target_ref, ref) + ) = _distance_and_filters( + expression, table_name, target_ref, ref, ctx.capabilities + ) where_sql = " AND ".join(where_clauses) # The wrapping level reads the inner row's *bare* column names (the passthrough # projected ``.*``), so the tiebreaker qualifies them by the wrapper @@ -358,7 +367,12 @@ def _fallback_form( where_clauses, passthrough, ) = _distance_and_filters( - expression, table_name, target_ref, ref, ref_fragments=ref_fragments + expression, + table_name, + target_ref, + ref, + ctx.capabilities, + ref_fragments=ref_fragments, ) # Surface the reference-key columns so the rewritten join can match each diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py index b0f3f79..a33480b 100644 --- a/src/giql/generators/base.py +++ b/src/giql/generators/base.py @@ -12,6 +12,7 @@ from giql.resolver import ResolvedRef from giql.table import Table from giql.table import Tables +from giql.targets import Capabilities class BaseGIQLGenerator(Generator): @@ -71,6 +72,7 @@ def _nearest_passthrough( target_start: str, target_end: str, output_table: Table | None, + capabilities: Capabilities | None = None, ) -> str: """Project the target's full row, de-canonicalizing the interval columns. @@ -80,10 +82,19 @@ def _nearest_passthrough( through as a plain ``{table_name}.*`` — the byte-identical identity fast path. When it is non-canonical the interval columns, canonical inside the ``__giql_canon_*`` CTE the target was rewritten to, are de-canonicalized - back into that encoding via a star ``REPLACE`` so the passed-through - interval matches the target's own convention. (Only non-canonical targets - are wrapped, so the ``REPLACE`` appears only where a canonical CTE already - shapes the SQL.) + back into that encoding so the passed-through interval matches the target's + own convention. + + The emit strategy is chosen from *capabilities*, following the precedent + of :func:`giql.expanders.disjoin._disjoin_passthrough` (issue #145) — the + same two emit forms, with an added ``capabilities is None`` arm for direct + callers (in production the sole caller always passes ``ctx.capabilities``): + + * ``{table_name}.* REPLACE (...)`` when ``supports_star_replace`` holds (or + no capabilities are supplied) — substitutes start/end in place; + * the portable ``{table_name}.* EXCEPT (start, end), , `` form + otherwise (the generic baseline / DataFusion family). Row-equivalent but + not column-order-equivalent, and not DuckDB-runnable. :param table_name: The relation the row is selected from (the canon CTE name when wrapped, @@ -94,9 +105,11 @@ def _nearest_passthrough( Physical end column name :param output_table: The target's declared :class:`~giql.table.Table`, or ``None`` + :param capabilities: + The active target's :class:`~giql.targets.Capabilities`; ``None`` + defaults to the ``* REPLACE`` form. :return: - The passthrough projection fragment (``{table_name}.*`` or a star - ``REPLACE``) + The passthrough projection fragment """ if output_table is None or ( output_table.coordinate_system == "0based" @@ -105,15 +118,16 @@ def _nearest_passthrough( return f"{table_name}.*" pt_start = decanonical_start(f'{table_name}."{target_start}"', output_table) pt_end = decanonical_end(f'{table_name}."{target_end}"', output_table) - # TODO(#142): this emits an unconditional ``* REPLACE`` (DuckDB-only). - # When DataFusion gains correlated LATERAL, adopt the capability branch the - # DISJOIN expander uses (``giql.expanders.disjoin._disjoin_passthrough``): - # ``* REPLACE`` where ``supports_star_replace`` holds, the portable - # ``* EXCEPT`` form otherwise, so a non-canonical NEAREST passthrough runs - # on the DataFusion family too. + if capabilities is None or capabilities.supports_star_replace: + return ( + f"{table_name}.* REPLACE " + f'({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' + ) + # Portable form for engines without ``* REPLACE`` (generic / DataFusion): + # drop the interval columns from the star and re-project them recomputed. return ( - f"{table_name}.* REPLACE " - f'({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' + f'{table_name}.* EXCEPT ("{target_start}", "{target_end}"), ' + f'{pt_start} AS "{target_start}", {pt_end} AS "{target_end}"' ) @staticmethod diff --git a/src/giql/targets.py b/src/giql/targets.py index 88e4eb7..940804b 100644 --- a/src/giql/targets.py +++ b/src/giql/targets.py @@ -6,11 +6,13 @@ This is step 1 of epic #137. The targets and capability descriptors defined here are the foundation that later steps build on — the operator-expander -registry is keyed by ``(target, operator)`` and emission choices become +registry is keyed by ``(target, operator)`` and emission choices are capability lookups rather than scattered ``if dialect == ...`` branches. -At this step the model is wired into :func:`giql.transpile.transpile` only -to resolve the ``dialect`` parameter and drive the existing DuckDB IEJoin -gate; no emission behaviour changes. +As of #143/#145 the model actively drives emission: the INTERSECTS join +strategy (``range_join_strategy``), the NEAREST LATERAL-vs-window form +(``supports_lateral``), and the coordinate-canonicalization wrapper and +NEAREST/DISJOIN passthroughs (``supports_star_replace``) all read from the +active target's capabilities. """ from dataclasses import dataclass @@ -37,9 +39,10 @@ class Capabilities: ``BaseGIQLGenerator.SUPPORTS_LATERAL`` generator attribute has been removed. supports_star_replace : bool - Whether the engine supports ``SELECT * REPLACE (...)``. Drives the - coordinate-canonicalization output: ``* REPLACE`` where supported, - an explicit portable projection otherwise (#143). Supported by + Whether the engine supports ``SELECT * REPLACE (...)``. Drives every + coordinate-canonicalization site — the canonicalizer wrapper CTE and the + DISJOIN / NEAREST passthroughs (#143/#145): ``* REPLACE`` where supported, + the portable ``* EXCEPT`` projection otherwise. Supported by DuckDB / BigQuery / Snowflake / ClickHouse; not by PostgreSQL, SQLite, or DataFusion. supports_qualify : bool @@ -86,14 +89,15 @@ class GenericTarget(Target): conservative, maximally portable baseline that matches today's :class:`giql.generators.base.BaseGIQLGenerator` output. - "SQL-92-ish", not strict SQL-92: because ``supports_star_replace=False``, the - DISJOIN passthrough over a **non-canonical** target falls back to a - ``SELECT * EXCEPT (...)`` projection (re-appending the de-canonicalized - interval columns). ``* EXCEPT`` is **not** SQL-92 and is **not + "SQL-92-ish", not strict SQL-92: because ``supports_star_replace=False``, + every coordinate-canonicalization site over a **non-canonical** target falls + back to a ``SELECT * EXCEPT (...)`` projection (re-appending the recomputed + interval columns) — the canonicalizer wrapper CTE and the DISJOIN / NEAREST + passthroughs alike. ``* EXCEPT`` is **not** SQL-92 and is **not DuckDB-runnable** — it is a DataFusion-family extension — so the generic - target's non-canonical DISJOIN output runs only on an ``* EXCEPT``-capable - engine. A canonical (0-based half-open) target passes the row through as a - plain, fully portable ``SELECT *``. + target's non-canonical output runs only on an ``* EXCEPT``-capable engine. + A canonical (0-based half-open) target passes the row through as a plain, + fully portable ``SELECT *``. """ name: str = "generic" @@ -128,12 +132,26 @@ class DuckDBTarget(Target): class DataFusionTarget(Target): """Apache DataFusion target. - sqlglot has no DataFusion dialect, so serialization falls back to the - generic form (``sqlglot_dialect = None``) for now; #145 finalizes - DataFusion serialization. The capability values below are conservative - and provisional — they are validated against a real DataFusion engine - when the operator migrations exercise them (#142, #145). DataFusion - supports ``* EXCEPT`` / ``* EXCLUDE`` but not ``* REPLACE``. + sqlglot has no DataFusion dialect, so serialization uses the generic form + (``sqlglot_dialect = None``); this is the finalized strategy (#145) — the + portable SQL the generic generator emits runs on DataFusion, verified + end-to-end by the cross-target oracle (``tests/integration/datafusion/``): + every operator at the default encoding, and the canonicalizing operators + (DISJOIN / NEAREST) across all four coordinate encodings plus custom-column + and strand schemas. + + The capability values below are validated against a real DataFusion engine by + that oracle: ``supports_lateral=False`` (no correlated-LATERAL physical plan, + so NEAREST takes the decorrelated window fallback), ``supports_star_replace= + False`` (DataFusion supports ``* EXCEPT`` / ``* EXCLUDE`` but not + ``* REPLACE``, so the canonicalizer and the NEAREST/DISJOIN passthroughs emit + the portable ``* EXCEPT`` form), ``supports_qualify=False``, and the binned + equi-join range strategy. + + One documented gap remains (#160, dependent on #146): a ``SELECT *`` / + ``SELECT b.*`` over a correlated NEAREST exposes the fallback's reserved + ``__giql_x_*`` columns, so the cross-target identity claim is narrowed to + explicitly-projected queries until a query-level projection seam lands. """ name: str = "datafusion" diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 0cd934f..2607ac9 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -55,8 +55,10 @@ def transpile( ) -> str: """Transpile a GIQL query to SQL. - Parses the GIQL syntax and converts it to standard SQL-92 compatible - output (uses LATERAL joins where needed for operations like NEAREST). + Parses the GIQL syntax and converts it to portable SQL for the active target + (uses LATERAL joins where needed for operations like NEAREST). The output is + not strictly SQL-92: depending on the target it may use engine extensions + such as ``LATERAL`` or ``SELECT * EXCEPT`` (see the ``dialect`` parameter). Parameters ---------- @@ -79,7 +81,15 @@ def transpile( ``"datafusion"`` and ``None`` use the generic binned equi-join path and accept ``intersects_bin_size``. Hard-error projection shapes raise ``ValueError`` at transpile time; see the performance guide - for the full enumeration. + for the full enumeration. The target's capabilities also choose the + coordinate-canonicalization emit form for a non-canonically-encoded + table: ``"duckdb"`` emits ``SELECT * REPLACE (...)``, while the generic + (``None``) and ``"datafusion"`` targets emit the portable + ``SELECT * EXCEPT (...), , `` form, which runs on + ``* EXCEPT``-capable engines (the DataFusion family) but **not** on + DuckDB — pass ``dialect="duckdb"`` for DuckDB-runnable output. Tables in + the canonical 0-based half-open encoding are unaffected (they emit + portable SQL on every target). intersects_bin_size : int | None Bin size for INTERSECTS equi-join optimization. When a query contains a full-table column-to-column INTERSECTS join, the @@ -235,9 +245,11 @@ def transpile( # Pass 2 of the normalization pipeline (epic #114): synthesize canonical # __giql_canon_* wrapper CTEs for non-canonical interval operands of operators # that opt in via GIQL_CANONICALIZE; those operators are rewritten here, and - # operators that do not opt in are left untouched. + # operators that do not opt in are left untouched. The active target's + # capabilities choose the wrapper's emit strategy (* REPLACE vs the portable + # * EXCEPT form — epic #137 / #145). with _reraise_as_value_error("Canonicalization error"): - ast = canonicalize_coordinates(ast) + ast = canonicalize_coordinates(ast, target.capabilities) # Pass 3 of the normalization pipeline (epic #137): replace each GIQL operator # node that opts in (GIQL_EXPAND) and resolves a registered expander with the diff --git a/tests/generators/test_base.py b/tests/generators/test_base.py index 3ac63c0..73e0d07 100644 --- a/tests/generators/test_base.py +++ b/tests/generators/test_base.py @@ -1659,37 +1659,45 @@ def test_giqlnearest_should_canonicalize_target_columns_for_each_convention( The query is transpiled. Then: It should wrap the target in a __giql_canon_* CTE carrying the - canonical conversion, and the distance CASE should read the bare - canonical target columns with no in-CASE canonicalization. + canonical conversion (a star REPLACE on DuckDB, the portable star + EXCEPT form on the generic / DataFusion family — #145), and the + distance CASE should read the bare canonical target columns with no + in-CASE canonicalization. """ # Arrange sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" + tables = [ + Table( + "genes", + coordinate_system=coordinate_system, + interval_type=interval_type, + ) + ] # Act — the target's coordinate canonicalization now lives in the # CanonicalizeCoordinates pass (#123): a non-canonical target is wrapped # in a __giql_canon_* CTE before generation, so the distance CASE reads - # already-canonical columns. - output = transpile( - sql, - tables=[ - Table( - "genes", - coordinate_system=coordinate_system, - interval_type=interval_type, - ) - ], - ) + # already-canonical columns. The wrapper's emit strategy is capability + # driven (#145): REPLACE on DuckDB, the portable EXCEPT form otherwise. + duckdb_output = transpile(sql, tables=tables, dialect="duckdb") + generic_output = transpile(sql, tables=tables) # Assert — the wrapper carries the canonical conversion; the distance # CASE reads the bare canonical columns against the literal [1000, 2000). - assert f"REPLACE ({wrap_start}, {wrap_end}) FROM genes" in output - assert ( - 'WHEN 1000 < __giql_canon_0."end" AND 2000 > __giql_canon_0."start" THEN 0' - ) in output + assert f"REPLACE ({wrap_start}, {wrap_end}) FROM genes" in duckdb_output assert ( - 'WHEN 2000 <= __giql_canon_0."start" THEN (__giql_canon_0."start" - 2000 + 1)' - ) in output - assert 'ELSE (1000 - __giql_canon_0."end" + 1)' in output + f'EXCEPT ("start", "end"), {wrap_start}, {wrap_end} FROM genes' + ) in generic_output + for output in (duckdb_output, generic_output): + assert ( + 'WHEN 1000 < __giql_canon_0."end" AND 2000 > __giql_canon_0."start" ' + "THEN 0" + ) in output + assert ( + 'WHEN 2000 <= __giql_canon_0."start" THEN ' + '(__giql_canon_0."start" - 2000 + 1)' + ) in output + assert 'ELSE (1000 - __giql_canon_0."end" + 1)' in output def test_giqlnearest_should_pass_target_columns_through_when_target_is_canonical( self, @@ -1724,25 +1732,31 @@ def test_giqlnearest_should_round_trip_passthrough_row_to_target_encoding(self): The query is transpiled. Then: The ``*`` passthrough should de-canonicalize the interval columns - back to the target's declared encoding via a star REPLACE, so the - returned row carries the table's own convention; the synthesized - ``distance`` column is encoding-invariant and stays unwrapped. + back to the target's declared encoding so the returned row carries the + table's own convention — via a star REPLACE on DuckDB and the portable + star EXCEPT form on the generic / DataFusion family (#145); the + synthesized ``distance`` column is encoding-invariant and stays + unwrapped. """ # Arrange sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" + tables = [Table("genes", coordinate_system="1based", interval_type="closed")] # Act - output = transpile( - sql, - tables=[Table("genes", coordinate_system="1based", interval_type="closed")], - ) + duckdb_output = transpile(sql, tables=tables, dialect="duckdb") + generic_output = transpile(sql, tables=tables) # Assert — passthrough de-canonicalizes 1-based-closed start as (start + 1) # and leaves end identity; the distance column carries no round-trip. assert ( '__giql_canon_0.* REPLACE ((__giql_canon_0."start" + 1) AS "start", ' '__giql_canon_0."end" AS "end")' - ) in output + ) in duckdb_output + assert ( + '__giql_canon_0.* EXCEPT ("start", "end"), ' + '(__giql_canon_0."start" + 1) AS "start", ' + '__giql_canon_0."end" AS "end"' + ) in generic_output def test_giqlnearest_should_canonicalize_reference_column_when_reference_is_one_based_closed( self, tables_mixed_conventions diff --git a/tests/integration/datafusion/test_cross_target_encodings.py b/tests/integration/datafusion/test_cross_target_encodings.py new file mode 100644 index 0000000..425b3d2 --- /dev/null +++ b/tests/integration/datafusion/test_cross_target_encodings.py @@ -0,0 +1,252 @@ +"""Cross-target coordinate-encoding sweep on the DataFusion engine (#145). + +The base ``cross_target_oracle`` lane (``test_cross_target_oracle.py``) runs every +operator at the default 0-based half-open encoding. Issue #145 extends the +DataFusion coverage across all four ``(coordinate_system, interval_type)`` +conventions and the custom-column / strand schema axes, so the capability-driven +canonicalization the canonicalizer and the NEAREST passthrough now perform +(``SELECT * REPLACE`` on DuckDB vs. the portable ``SELECT * EXCEPT`` form on the +generic / DataFusion family) is actually executed on the real DataFusion engine +for every encoding — not just proven byte-shaped in the unit suite. + +For DISJOIN the full three-target oracle runs: the generic and datafusion targets +emit the portable ``* EXCEPT`` canonical wrapper and passthrough, both executed on +DataFusion, while duckdb runs the ``* REPLACE`` form on DuckDB. + +For NEAREST the sweep restricts to ``("datafusion", "duckdb")``. A non-canonical +correlated NEAREST has no runnable generic target: the generic capability set is +lateral-capable (so it emits the ``LATERAL`` form, which DataFusion cannot plan) +yet not ``* REPLACE``-capable (so it emits ``* EXCEPT``, which DuckDB cannot run). +The datafusion target (decorrelated fallback + ``* EXCEPT``, on DataFusion) and the +duckdb target (``LATERAL`` + ``* REPLACE``, on DuckDB) still pin cross-engine +identity for every encoding. +""" + +import pytest + +pytest.importorskip("duckdb") +pytest.importorskip("datafusion") +pytest.importorskip("pyarrow") + +from giql import Table # noqa: E402 + +from ..coordinate_space.encodings import CONVENTIONS # noqa: E402 +from ..coordinate_space.encodings import encode # noqa: E402 +from ..coordinate_space.encodings import make_table # noqa: E402 + +pytestmark = pytest.mark.integration + + +def _interval(canonical_start, canonical_end, coordinate_system, interval_type): + """Encode a canonical interval into a ``(start, end)`` tuple for a convention.""" + return encode(canonical_start, canonical_end, coordinate_system, interval_type) + + +class TestDisjoinEncodingSweepOnDataFusion: + """DISJOIN canonical-wrapper + passthrough across all encodings on DataFusion.""" + + @pytest.mark.parametrize(("coordinate_system", "interval_type"), CONVENTIONS) + def test_disjoin_sweep_agrees_across_targets( + self, cross_target_oracle, coordinate_system, interval_type + ): + """Test DISJOIN agrees across targets for every coordinate encoding. + + Given: + Two overlapping intervals stored under one of the four + (coordinate_system, interval_type) conventions, declared on a Table + with that encoding. + When: + The DISJOIN oracle runs all three targets — generic and datafusion + emit the portable EXCEPT canonical wrapper and passthrough (executed + on DataFusion), duckdb emits the REPLACE form (executed on DuckDB). + Then: + Every target should return the same sub-segments, with the + passthrough and disjoin columns de-canonicalized back to the declared + encoding, proving the EXCEPT canonicalization runs on DataFusion for + every encoding. + """ + # Arrange + cs, it = coordinate_system, interval_type + feats = [ + ("chr1",) + _interval(0, 100, cs, it), + ("chr1",) + _interval(50, 150, cs, it), + ] + canonical_rows = [ + ((0, 100), (0, 50)), + ((0, 100), (50, 100)), + ((50, 150), (50, 100)), + ((50, 150), (100, 150)), + ] + expected = [ + _interval(*parent, cs, it) + _interval(*sub, cs, it) + for parent, sub in canonical_rows + ] + + # Act & assert + cross_target_oracle( + 'SELECT start, "end", disjoin_start, disjoin_end FROM DISJOIN(feats)', + tables=[make_table("feats", cs, it)], + feats=feats, + expected=expected, + ) + + +class TestNearestEncodingSweepOnDataFusion: + """NEAREST canonical-wrapper + passthrough across encodings on DataFusion.""" + + @pytest.mark.parametrize(("coordinate_system", "interval_type"), CONVENTIONS) + def test_correlated_nearest_non_canonical_target_agrees_datafusion_vs_duckdb( + self, cross_target_oracle, coordinate_system, interval_type + ): + """Test correlated NEAREST over a non-canonical target agrees per encoding. + + Given: + A canonical peaks table and a candidate genes table stored under one + of the four conventions, declared with that encoding. + When: + A correlated NEAREST k=1 runs on the datafusion target (decorrelated + fallback + portable EXCEPT passthrough, on DataFusion) and the duckdb + target (LATERAL + REPLACE, on DuckDB). + Then: + Both should return the same nearest gene with its interval columns + de-canonicalized back to the declared encoding and an + encoding-invariant distance, proving the EXCEPT canonical wrapper and + passthrough run on DataFusion for every encoding. + """ + # Arrange + cs, it = coordinate_system, interval_type + genes = [ + ("chr1",) + _interval(1000, 1100, cs, it), + ("chr1",) + _interval(50, 60, cs, it), + ("chr1",) + _interval(280, 290, cs, it), + ] + nearest_start, nearest_end = _interval(280, 290, cs, it) + + # Act & assert + cross_target_oracle( + "SELECT b.start AS b_start, b.\"end\" AS b_end, distance " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + tables=[Table("peaks"), make_table("genes", cs, it)], + peaks=[("chr1", 200, 300)], + genes=genes, + expected=[(nearest_start, nearest_end, 0)], + targets=("datafusion", "duckdb"), + ) + + + def test_correlated_nearest_custom_columns_agree_datafusion_vs_duckdb( + self, cross_target_oracle + ): + """Test correlated NEAREST over a non-canonical custom-column target agrees. + + Given: + A peaks table and a non-canonical (1based/closed) genes table that both + name their interval columns gc / gs / ge rather than chrom/start/end. + When: + A correlated NEAREST k=1 runs on the datafusion target (decorrelated + fallback + EXCEPT passthrough over the custom names) and the duckdb + target (LATERAL + REPLACE). + Then: + Both should return the same nearest gene with its custom-named interval + columns de-canonicalized, proving the EXCEPT NEAREST passthrough + resolves custom column names on DataFusion (the DISJOIN custom-column + path's NEAREST counterpart). + """ + # Arrange — both tables share one schema (the oracle registers all table + # data under a single `columns` mapping); only genes is non-canonical. + columns = (("gc", "utf8"), ("gs", "int64"), ("ge", "int64")) + peaks = Table("peaks", chrom_col="gc", start_col="gs", end_col="ge") + genes = Table( + "genes", + chrom_col="gc", + start_col="gs", + end_col="ge", + coordinate_system="1based", + interval_type="closed", + ) + nearest_start, nearest_end = encode(280, 290, "1based", "closed") + + # Act & assert + cross_target_oracle( + "SELECT b.gs AS b_start, b.ge AS b_end, distance " + "FROM peaks a " + "CROSS JOIN LATERAL NEAREST(genes, reference := a.interval, k := 1) b", + tables=[peaks, genes], + columns=columns, + peaks=[("chr1", 200, 300)], + genes=[ + ("chr1",) + encode(1000, 1100, "1based", "closed"), + ("chr1",) + encode(50, 60, "1based", "closed"), + ("chr1",) + encode(280, 290, "1based", "closed"), + ], + expected=[(nearest_start, nearest_end, 0)], + targets=("datafusion", "duckdb"), + ) + + +class TestDataFusionSchemaAxes: + """Custom column names and strand schemas exercised on DataFusion (#145).""" + + def test_disjoin_custom_column_names_agree_across_targets( + self, cross_target_oracle + ): + """Test DISJOIN over a custom-column table agrees across targets. + + Given: + Two overlapping intervals in a table whose interval columns are named + ch / s / e rather than the canonical chrom / start / end. + When: + The DISJOIN oracle runs all three targets. + Then: + Every target should resolve the custom column names and return the + same sub-segments, proving custom-column resolution runs on DataFusion. + """ + # Arrange / Act / Assert + cross_target_oracle( + "SELECT s, e, disjoin_start, disjoin_end FROM DISJOIN(feats)", + tables=[Table("feats", chrom_col="ch", start_col="s", end_col="e")], + columns=(("ch", "utf8"), ("s", "int64"), ("e", "int64")), + feats=[("chr1", 0, 100), ("chr1", 50, 150)], + expected=[ + (0, 100, 0, 50), + (0, 100, 50, 100), + (50, 150, 50, 100), + (50, 150, 100, 150), + ], + ) + + def test_stranded_merge_agrees_across_targets(self, cross_target_oracle): + """Test stranded MERGE aggregates within strand on every target. + + Given: + Overlapping intervals on chr1 split across the + and - strands. + When: + A stranded MERGE runs on all three targets (generic and datafusion on + DataFusion, duckdb on DuckDB). + Then: + Every target should merge only same-strand overlaps, yielding one + merged interval per strand, and agree — exercising the strand schema + axis on DataFusion. + """ + # Arrange / Act / Assert — MERGE rewrites the whole SELECT, projecting + # chrom, strand, MIN(start) AS start, MAX(end) AS end. + cross_target_oracle( + "SELECT MERGE(interval, stranded := true) FROM feats", + tables=[Table("feats", strand_col="strand")], + columns=( + ("chrom", "utf8"), + ("start", "int64"), + ("end", "int64"), + ("strand", "utf8"), + ), + feats=[ + ("chr1", 0, 100, "+"), + ("chr1", 50, 150, "+"), + ("chr1", 0, 100, "-"), + ], + expected=[ + ("chr1", "+", 0, 150), + ("chr1", "-", 0, 100), + ], + ) diff --git a/tests/test_canonicalizer.py b/tests/test_canonicalizer.py index 57b751c..d017913 100644 --- a/tests/test_canonicalizer.py +++ b/tests/test_canonicalizer.py @@ -31,6 +31,7 @@ from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables +from giql.targets import DuckDBTarget from giql.targets import GenericTarget from giql.transpile import transpile @@ -66,6 +67,11 @@ def _prepare(query: str, tables: Tables) -> exp.Expression: return canonicalize_coordinates(ast) +def _resolve(query: str, tables: Tables) -> exp.Expression: + """Parse *query* and run pass 1 only, leaving pass 2 for the caller.""" + return resolve_operator_refs(parse_one(query, dialect=GIQLDialect), tables) + + def _canon_ctes(ast: exp.Expression) -> list[exp.CTE]: """Return every synthesized canonical CTE in *ast*, in declaration order.""" with_ = ast.args.get("with_") @@ -788,23 +794,30 @@ def test_non_canonical_target_wrapped_and_row_round_tripped(self): Then: The target should be wrapped in a __giql_canon_* CTE, the distance CASE should read the bare canonical columns, and the passed-through - row should de-canonicalize the interval back to the declared encoding. + row should de-canonicalize the interval back to the declared encoding + — via a star REPLACE on DuckDB and the portable star EXCEPT form on + the generic / DataFusion family (#145). """ # Arrange sql = "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 1)" + tables = [Table("genes", coordinate_system="1based", interval_type="closed")] # Act - output = transpile( - sql, - tables=[Table("genes", coordinate_system="1based", interval_type="closed")], - ) + duckdb_output = transpile(sql, tables=tables, dialect="duckdb") + generic_output = transpile(sql, tables=tables) # Assert - assert f"{CANON_PREFIX}0 AS (SELECT * REPLACE" in output - assert 'WHEN 1000 < __giql_canon_0."end"' in output + assert f"{CANON_PREFIX}0 AS (SELECT * REPLACE" in duckdb_output + assert f"{CANON_PREFIX}0 AS (SELECT * EXCEPT" in generic_output + for output in (duckdb_output, generic_output): + assert 'WHEN 1000 < __giql_canon_0."end"' in output assert ( '__giql_canon_0.* REPLACE ((__giql_canon_0."start" + 1) AS "start"' - ) in output + ) in duckdb_output + assert ( + '__giql_canon_0.* EXCEPT ("start", "end"), ' + '(__giql_canon_0."start" + 1) AS "start"' + ) in generic_output def test_canonical_target_not_wrapped(self): """Test a canonical NEAREST target is left unwrapped. @@ -834,3 +847,155 @@ def test_canonical_target_not_wrapped(self): # Assert assert opted_in == flag_off assert CANON_PREFIX not in opted_in + + +class TestCanonicalProjectionCapabilities: + """The wrapper-CTE emit strategy is chosen from target capabilities (#145).""" + + def test_canonicalize_coordinates_should_emit_except_without_star_replace( + self, disjoin_opted_in + ): + """Test the wrapper uses the portable EXCEPT form without star-replace support. + + Given: + A non-canonical DISJOIN AST and a Capabilities lacking + supports_star_replace (the generic / DataFusion family). + When: + canonicalize_coordinates is called directly with those capabilities. + Then: + The synthesized wrapper CTE should project the portable + SELECT * EXCEPT (...) form and never a star REPLACE. + """ + # Arrange + ast = _resolve("SELECT * FROM DISJOIN(variants)", _tables(("1based", "closed"))) + + # Act + canonicalize_coordinates(ast, GenericTarget().capabilities) + + # Assert + body = _canon_ctes(ast)[0].this.sql() + assert "* EXCEPT" in body + assert "REPLACE" not in body + assert '("start" - 1) AS "start"' in body + + def test_canonicalize_coordinates_should_emit_replace_with_star_replace( + self, disjoin_opted_in + ): + """Test the wrapper uses star REPLACE on a REPLACE-capable target. + + Given: + A non-canonical DISJOIN AST and a Capabilities with + supports_star_replace (DuckDB). + When: + canonicalize_coordinates is called directly with those capabilities. + Then: + The synthesized wrapper CTE should substitute the interval columns in + place via a star REPLACE and never the EXCEPT form. + """ + # Arrange + ast = _resolve("SELECT * FROM DISJOIN(variants)", _tables(("1based", "closed"))) + + # Act + canonicalize_coordinates(ast, DuckDBTarget().capabilities) + + # Assert + body = _canon_ctes(ast)[0].this.sql() + assert "* REPLACE" in body + assert "EXCEPT" not in body + assert '("start" - 1) AS "start"' in body + + def test_canonicalize_coordinates_should_emit_replace_when_capabilities_none( + self, disjoin_opted_in + ): + """Test a direct caller with no capabilities keeps the historical REPLACE form. + + Given: + A non-canonical DISJOIN AST. + When: + canonicalize_coordinates is called with no capabilities argument. + Then: + The wrapper should default to the star REPLACE form, preserving the + historical behavior for direct callers. + """ + # Arrange + ast = _resolve("SELECT * FROM DISJOIN(variants)", _tables(("1based", "closed"))) + + # Act + canonicalize_coordinates(ast) + + # Assert + body = _canon_ctes(ast)[0].this.sql() + assert "* REPLACE" in body + assert "EXCEPT" not in body + + def test_canonicalize_coordinates_should_not_re_wrap_on_a_second_pass( + self, disjoin_opted_in + ): + """Test the pass is idempotent over the EXCEPT wrapper it already inserted. + + Given: + A non-canonical DISJOIN AST canonicalized once with EXCEPT-form + capabilities. + When: + canonicalize_coordinates is run a second time over the same AST. + Then: + It should not synthesize a second wrapper — the rewritten slot now + points at the canonical CTE, so exactly one __giql_canon_ CTE remains. + """ + # Arrange + caps = GenericTarget().capabilities + ast = _resolve("SELECT * FROM DISJOIN(variants)", _tables(("1based", "closed"))) + canonicalize_coordinates(ast, caps) + + # Act + canonicalize_coordinates(ast, caps) + + # Assert + assert len(_canon_ctes(ast)) == 1 + + def test_transpile_should_emit_except_wrapper_for_datafusion_dialect(self): + """Test the DataFusion dialect threads its capabilities into the wrapper. + + Given: + A non-canonical DISJOIN over a 1-based closed table. + When: + Transpiling with dialect="datafusion". + Then: + The wrapper CTE and passthrough should use the portable EXCEPT form + (DataFusion lacks star REPLACE) with no leaked operator. + """ + # Arrange + tables = [Table("variants", coordinate_system="1based", interval_type="closed")] + + # Act + sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=tables, dialect="datafusion" + ) + + # Assert + assert "SELECT * EXCEPT" in sql + assert "REPLACE" not in sql + assert "G_I_Q_L" not in sql + + def test_transpile_should_emit_except_wrapper_when_no_dialect(self): + """Test the generic (no-dialect) default emits the portable EXCEPT wrapper. + + Given: + A non-canonical DISJOIN over a 1-based closed table. + When: + Transpiling with no dialect (the generic default target). + Then: + The wrapper CTE should use the portable EXCEPT form, not star REPLACE + — pinning that the no-dialect default resolves GenericTarget + (supports_star_replace=False) and emits the not-DuckDB-runnable + portable form, guarding against a regression back to REPLACE. + """ + # Arrange + tables = [Table("variants", coordinate_system="1based", interval_type="closed")] + + # Act + sql = transpile("SELECT * FROM DISJOIN(variants)", tables=tables) + + # Assert + assert "SELECT * EXCEPT" in sql + assert "REPLACE" not in sql diff --git a/tests/test_disjoin_transpilation.py b/tests/test_disjoin_transpilation.py index 426adf0..9bd67e2 100644 --- a/tests/test_disjoin_transpilation.py +++ b/tests/test_disjoin_transpilation.py @@ -503,6 +503,7 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_explicit_self_reference_ interval_type="closed", ) ], + dialect="duckdb", ) # Assert @@ -511,6 +512,8 @@ def test_giqldisjoin_sql_should_skip_exists_clause_when_explicit_self_reference_ # Target and explicit self-reference dedupe to a single wrapper CTE: one # CTE definition plus the two FROM references (target + reference). assert sql.count("__giql_canon_") == 3 + # REPLACE is the DuckDB wrapper form; the generic / DataFusion target emits + # the portable EXCEPT form (#145), covered separately. assert "SELECT * REPLACE" in sql def test_giqldisjoin_sql_should_emit_one_exists_clause_when_query_mixes_self_and_distinct_reference_disjoins( @@ -752,10 +755,13 @@ def test_giqldisjoin_sql_should_wrap_non_canonical_target_in_canon_cte(self): tables=[ Table("features", coordinate_system="1based", interval_type="closed") ], + dialect="duckdb", ) # Assert assert "__giql_canon_" in sql + # REPLACE is the DuckDB wrapper form; generic / DataFusion emits the + # portable EXCEPT form (#145). assert "SELECT * REPLACE" in sql assert '("start" - 1) AS "start"' in sql # No inline canonicalization arithmetic survives in the emitter output. @@ -781,10 +787,13 @@ def test_giqldisjoin_sql_should_wrap_non_canonical_reference_in_canon_cte(self): "features", Table("refs", coordinate_system="1based", interval_type="closed"), ], + dialect="duckdb", ) # Assert assert "__giql_canon_" in sql + # REPLACE is the DuckDB wrapper form; generic / DataFusion emits the + # portable EXCEPT form (#145). assert "SELECT * REPLACE" in sql # The breakpoint CTE reads the canonical endpoints verbatim, no inline shift. assert '("start" - 1) AS pos' not in sql @@ -827,9 +836,11 @@ def test_giqldisjoin_sql_should_dedupe_self_reference_to_one_canon_cte(self): tables=[ Table("features", coordinate_system="1based", interval_type="closed") ], + dialect="duckdb", ) - # Assert + # Assert — REPLACE is the DuckDB wrapper form (#145); the dedupe behavior + # is target-independent. assert sql.count("AS (SELECT * REPLACE") == 1 assert sql.count("__giql_canon_") == 3 From 3b92c43984d1370b4def94c353a17f9d864d66a9 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 1 Jul 2026 17:07:05 -0400 Subject: [PATCH 099/142] refactor: Reduce the generator to a stock per-target serializer Every GIQL operator now expands to standard AST in the normalization passes, so the custom generator earns its keep no longer. Delete the giql.generators package and BaseGIQLGenerator and serialize the final AST with the stock sqlglot serializer selected by the active target's sqlglot_dialect. The five NEAREST helpers and the distance-CASE string builder move out of the generator into the expander modules. Remove the GIQL_EXPAND migration flag. With every operator migrated the per-node opt-in gate is dead weight, so it and the legacy emit path are gone; a resolve miss now raises a clear error instead of falling back to a legacy emitter that no longer exists. Make the expander registry a target plugin hub. A custom Target can be declared with register_target, or as a side effect of register, and selected with transpile(dialect=name), so the registry becomes a supported public extension point. resolve_target consults the registry for a non-built-in name while built-in names still win. Export the hook symbols from the top-level giql package and document the extension point with a runnable worked example. Serialization is byte-identical to the previous generator for the generic and DataFusion targets. The only change is that the DuckDB target now makes DuckDB's default NULLS FIRST ordering explicit. Claude-Session: https://claude.ai/code/session_01ALxmQysPad4W68wuWuft6W --- docs/index.rst | 1 + docs/transpilation/api-reference.rst | 47 +- docs/transpilation/extending.rst | 171 ++++ docs/transpilation/schema-mapping.rst | 5 + src/giql/__init__.py | 21 + src/giql/expander.py | 281 +++--- src/giql/expanders/_distance.py | 117 +++ src/giql/expanders/_params.py | 37 + src/giql/expanders/disjoin.py | 10 +- src/giql/expanders/distance.py | 54 +- src/giql/expanders/intersects.py | 12 +- src/giql/expanders/nearest.py | 178 +++- src/giql/expressions.py | 53 +- src/giql/generators/__init__.py | 5 - src/giql/generators/base.py | 339 -------- src/giql/resolver.py | 4 +- src/giql/targets.py | 48 +- src/giql/transpile.py | 48 +- tests/generators/__init__.py | 0 .../integration/coordinate_space/encodings.py | 5 +- tests/test_canonicalizer.py | 5 +- tests/test_distance_transpilation.py | 3 +- tests/test_distance_udf.py | 42 +- tests/test_expander.py | 817 +++++++++--------- tests/test_nearest_transpilation.py | 40 +- tests/test_package_api.py | 95 ++ .../test_base.py => test_serialization.py} | 121 +-- tests/test_targets.py | 273 +++++- tests/test_transpile.py | 46 + 29 files changed, 1726 insertions(+), 1152 deletions(-) create mode 100644 docs/transpilation/extending.rst create mode 100644 src/giql/expanders/_distance.py create mode 100644 src/giql/expanders/_params.py delete mode 100644 src/giql/generators/__init__.py delete mode 100644 src/giql/generators/base.py delete mode 100644 tests/generators/__init__.py create mode 100644 tests/test_package_api.py rename tests/{generators/test_base.py => test_serialization.py} (94%) diff --git a/docs/index.rst b/docs/index.rst index 71237d6..78af604 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,6 +48,7 @@ See the :doc:`GIQL transpiler ` docs. transpilation/schema-mapping transpilation/execution transpilation/performance + transpilation/extending transpilation/api-reference diff --git a/docs/transpilation/api-reference.rst b/docs/transpilation/api-reference.rst index fcba984..ca771a3 100644 --- a/docs/transpilation/api-reference.rst +++ b/docs/transpilation/api-reference.rst @@ -1,13 +1,48 @@ API Reference ============= -.. currentmodule:: giql +Transpilation +------------- -.. autosummary:: +.. autofunction:: giql.transpile.transpile - transpile - Table +.. autoclass:: giql.table.Table -.. autofunction:: transpile +Extension hook +-------------- -.. autoclass:: Table +The registry-based extension point for custom targets and operator expanders +(see :doc:`extending`). Every symbol below is re-exported from the top-level +``giql`` package. + +.. autofunction:: giql.expander.register + +.. autodata:: giql.expander.REGISTRY + +.. autoclass:: giql.expander.ExpanderRegistry + :members: register, register_target, target, resolve, has_override, + unregister, clear, snapshot, restore + +.. autoclass:: giql.expander.RegistrySnapshot + :members: + +.. autoclass:: giql.expander.ExpansionContext + :members: + +.. autoclass:: giql.expander.OperatorExpander + :members: + +Targets and capabilities +------------------------ + +.. autoclass:: giql.targets.Target + :members: + +.. autoclass:: giql.targets.Capabilities + :members: + +.. autoclass:: giql.targets.GenericTarget + +.. autoclass:: giql.targets.DuckDBTarget + +.. autoclass:: giql.targets.DataFusionTarget diff --git a/docs/transpilation/extending.rst b/docs/transpilation/extending.rst new file mode 100644 index 0000000..9df4c7c --- /dev/null +++ b/docs/transpilation/extending.rst @@ -0,0 +1,171 @@ +Extending GIQL: custom targets and operators +============================================= + +Every GIQL operator is expanded to standard SQL by a small **expander** function +selected from a process-wide registry, keyed by ``(target, operator)``. That +registry is a supported public extension point. Through it you can: + +- **register a custom target** — a new SQL engine with its own capabilities and + sqlglot output dialect — and select it with ``transpile(dialect="")``; and +- **override an operator's expander** for a target, so that operator emits SQL + tailored to that engine. + +Both hooks live in the ``giql.expander`` and ``giql.targets`` modules, and the key +symbols are re-exported from the top-level package: + +.. code-block:: python + + from giql import ( + transpile, + register, # decorator: register/override an operator expander + REGISTRY, # the process-wide plugin registry + Target, # base class for a custom target + Capabilities, # a target's feature set + GenericTarget, # the portable fallback target + ExpansionContext, + ) + + +Targets and capabilities +------------------------ + +A :class:`~giql.targets.Target` is a first-class SQL engine. It carries a +``name``, the ``sqlglot_dialect`` used to serialize the final AST (``None`` +selects sqlglot's portable default), and a :class:`~giql.targets.Capabilities` +set. Capabilities — not scattered ``if dialect == ...`` branches — drive the +portable emission choices: + +- ``supports_lateral`` — correlated ``LATERAL`` joins (NEAREST uses a decorrelated + window-function form when this is ``False``); +- ``supports_star_replace`` — ``SELECT * REPLACE (...)`` (used for coordinate + canonicalization and the DISJOIN / NEAREST passthroughs; the portable + ``SELECT * EXCEPT (...)`` form is emitted otherwise); +- ``supports_qualify`` — the ``QUALIFY`` clause; +- ``range_join_strategy`` — ``"binned"`` or ``"iejoin"`` for column-to-column + INTERSECTS joins. + +Define a custom target by subclassing :class:`~giql.targets.Target` as a frozen +dataclass. Give every field a default so the class is constructible with no +arguments (the ``@register`` decorator instantiates a target class for you): + +.. code-block:: python + + from dataclasses import dataclass + from giql import Target, Capabilities + + @dataclass(frozen=True) + class PostgresTarget(Target): + name: str = "postgres" + sqlglot_dialect: str | None = "postgres" + capabilities: Capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=False, # -> portable "* EXCEPT" canonicalization + supports_qualify=True, + range_join_strategy="binned", + ) + + +Registering and selecting a custom target +----------------------------------------- + +Declare the target on the registry, then select it by name. A **capability-only** +target — one that reuses every built-in operator expander and differs only in its +capabilities and dialect — needs nothing more than +:meth:`~giql.expander.ExpanderRegistry.register_target`: + +.. code-block:: python + + from giql import REGISTRY, transpile + + REGISTRY.register_target(PostgresTarget()) + + sql = transpile( + "SELECT * FROM peaks WHERE interval WITHIN 'chr1:1000-5000'", + tables=["peaks"], + dialect="postgres", + ) + +``dialect="postgres"`` resolves the registered target; its +``supports_star_replace=False`` capability selects the portable projection form, +and its ``sqlglot_dialect="postgres"`` serializes the final SQL. (The built-in +names ``"duckdb"`` and ``"datafusion"`` and ``None`` for the generic target still +resolve as before; ``"generic"`` is *not* a selectable name — ``None`` is the +sole way to select the generic target.) + + +Overriding an operator's expander +--------------------------------- + +To tailor one operator to an engine, register an expander for +``(target, operator)`` with the :func:`~giql.expander.register` decorator. An +expander takes the operator node and its :class:`~giql.expander.ExpansionContext` +and returns the sqlglot AST that replaces the node: + +.. code-block:: python + + from sqlglot import parse_one + from giql import register, ExpansionContext + from giql.expressions import Within + + @register(PostgresTarget, Within) + def within_between(node, ctx: ExpansionContext): + # ctx.resolution exposes the operator's resolved column operands, and + # ctx.capabilities / ctx.target the active engine's capabilities. + col = ctx.resolution.column("this") + lo, hi = 1000, 5000 # (a real expander parses the node's range) + return parse_one(f"{col.start} BETWEEN {lo} AND {hi}") + +Registering an expander also **declares its target by name** as a side effect, so +``@register(PostgresTarget, Within)`` alone makes ``dialect="postgres"`` +resolvable — a separate ``register_target`` call is only needed for a target that +overrides no operators. + +Resolution follows a fallback chain: an exact ``(target, operator)`` expander wins, +otherwise the built-in ``(GenericTarget(), operator)`` expander runs. So you only +register the operators that genuinely differ for your engine; everything else +reuses the portable built-ins. + +.. note:: + + ``ctx.resolution.column("this").start`` (and ``.end`` / ``.chrom``) return + **quoted** physical column names (e.g. ``"start"``, or ``a."start"`` when the + operand is aliased), ready to splice into a SQL string that you ``parse_one``. + Do not pass them to ``sqlglot.exp.column()``, + which would quote them a second time. Building fragments as strings and parsing + once — as the built-in expanders do — is the simplest correct approach. + + +The node-local boundary +----------------------- + +An expander's **return value** is node-local: ``expand(node, ctx)`` returns the +one expression that replaces the operator node in place. It cannot *return* a +reshaped enclosing query. An expander may still restructure the query it sits in +as a side effect and then return the node unchanged — the built-in CLUSTER and +MERGE expanders do exactly this, rewriting their single-table ``SELECT`` in place. +What no expander can express is a rewrite that **adds or reshapes joins** across +relations: the DuckDB IEJoin plan for column-to-column INTERSECTS joins is handled +by a capability-gated pre-pass transformer, not an expander, because it restructures +the surrounding join. A general query-level expander seam for such join rewrites is +planned future work. + + +Undoing a registration +----------------------- + +The registry exposes a teardown seam so a plugin or a test fixture can undo its +registrations rather than reaching into private state: + +.. code-block:: python + + REGISTRY.unregister(PostgresTarget(), Within) # drop one expander entry + REGISTRY.clear() # drop all entries and targets + + saved = REGISTRY.snapshot() # save/restore around a body + try: + ... # register custom entries + finally: + REGISTRY.restore(saved) + +See :func:`giql.transpile.transpile`, :func:`giql.expander.register`, and +:class:`giql.targets.Capabilities` in the :doc:`api-reference`. diff --git a/docs/transpilation/schema-mapping.rst b/docs/transpilation/schema-mapping.rst index b991e45..fd344b0 100644 --- a/docs/transpilation/schema-mapping.rst +++ b/docs/transpilation/schema-mapping.rst @@ -198,6 +198,11 @@ If your data uses 1-based coordinates (like VCF or GFF), configure the at transpile time and raises a ``ValueError`` naming the missing column(s) rather than emitting SQL that fails with an engine ``column not found`` error. + The projection form is chosen from the active target's + :class:`~giql.targets.Capabilities` (``supports_star_replace``). A custom + engine can select either form by registering a target with the appropriate + capabilities -- see :doc:`extending`. + Working with Point Features ~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/giql/__init__.py b/src/giql/__init__.py index e5df351..a2dce2d 100644 --- a/src/giql/__init__.py +++ b/src/giql/__init__.py @@ -4,7 +4,17 @@ """ from giql.constants import DEFAULT_BIN_SIZE +from giql.expander import REGISTRY +from giql.expander import ExpanderRegistry +from giql.expander import ExpansionContext +from giql.expander import OperatorExpander +from giql.expander import register from giql.table import Table +from giql.targets import Capabilities +from giql.targets import DataFusionTarget +from giql.targets import DuckDBTarget +from giql.targets import GenericTarget +from giql.targets import Target from giql.transpile import transpile __version__ = "0.1.0" @@ -14,4 +24,15 @@ "DEFAULT_BIN_SIZE", "Table", "transpile", + # Extension hook: register custom targets and override operator expanders. + "register", + "REGISTRY", + "ExpanderRegistry", + "ExpansionContext", + "OperatorExpander", + "Target", + "Capabilities", + "GenericTarget", + "DuckDBTarget", + "DataFusionTarget", ] diff --git a/src/giql/expander.py b/src/giql/expander.py index bab00fb..9809d7b 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -1,7 +1,7 @@ """The operator-expander registry and the ``ExpandOperators`` pass (epic #137). -This module is step 2 of epic #137. It lands the dispatch infrastructure that the -remaining steps migrate each operator onto, one at a time: +This module provides the dispatch infrastructure every GIQL operator expands +through (epic #137, complete): * an :class:`OperatorExpander` protocol — ``expand(node, ctx) -> exp.Expression``, the unit that turns one GIQL operator node into standard sqlglot AST for the @@ -11,8 +11,7 @@ :class:`~giql.targets.Target` and its :class:`~giql.targets.Capabilities`, collision-safe alias minting, and the registered :class:`~giql.table.Tables`; * a :class:`ExpanderRegistry` keyed by ``(target, operator_type)`` resolving an - expander through the fallback chain ``(target, op)`` → ``(generic, op)`` → - the legacy ``*_sql`` emitter; + expander through the fallback chain ``(target, op)`` → ``(generic, op)``; * a :func:`register` decorator — the **public extension hook** by which a user adds a target or overrides an operator for one, e.g. ``@register(DuckDBTarget, GIQLDisjoin)``; @@ -26,29 +25,25 @@ ``(DuckDBTarget(), GIQLDisjoin) in REGISTRY`` tests for one; * the :class:`ExpandOperators` pass, which runs after :func:`giql.canonicalizer.canonicalize_coordinates`, walks the AST, and - replaces every opted-in GIQL operator node with the registry's expansion. - -Gating (mirrors ``GIQL_CANONICALIZE``) --------------------------------------- -The pass is gated per operator by a ``GIQL_EXPAND`` class attribute on the -operator's expression class, exactly as pass 2 is gated by ``GIQL_CANONICALIZE``. -An operator takes the new AST-expansion path **only when both** hold: - -* its class sets ``GIQL_EXPAND = True``, and -* the registry resolves an expander for ``(active target, operator type)`` — - i.e. a ``(target, op)`` or ``(generic, op)`` expander is registered. - -Otherwise it falls through to the legacy ``*_sql`` emitter on -:class:`giql.generators.base.BaseGIQLGenerator`. The built-in expanders register -at import time via :mod:`giql.expanders`; the pass rewrites a node only when -``GIQL_EXPAND=True`` **and** an expander resolves for ``(active target, operator -type)``, and is a no-op for any operator that is unflagged or has no registered -expander. A migration PR registers an expander, flips one operator's -``GIQL_EXPAND`` flag, and deletes that operator's ``*_sql`` method. + replaces every GIQL operator node with the registry's expansion. + +Dispatch +-------- +With every operator migrated (epic #137 complete), the pass rewrites *every* GIQL +operator node: it resolves an expander for ``(active target, operator type)`` +through the registry's fallback chain and replaces the node with the AST that +expander returns. The built-in expanders register at import time via +:mod:`giql.expanders`, so a built-in operator always resolves at least a +``(generic, op)`` expander; a resolution miss is therefore an internal invariant +violation (an operator with no registered expander) and the pass raises rather +than leaving an un-serializable GIQL node in the tree. There is no per-operator +opt-in flag and no legacy ``*_sql`` fallback — both were migration scaffolding +removed once every operator was migrated (#146). """ from __future__ import annotations +from dataclasses import dataclass from typing import Callable from typing import Protocol from typing import runtime_checkable @@ -56,6 +51,15 @@ from sqlglot import exp from sqlglot.helper import name_sequence +from giql.expressions import Contains +from giql.expressions import GIQLCluster +from giql.expressions import GIQLDisjoin +from giql.expressions import GIQLDistance +from giql.expressions import GIQLMerge +from giql.expressions import GIQLNearest +from giql.expressions import Intersects +from giql.expressions import SpatialSetPredicate +from giql.expressions import Within from giql.resolver import META_KEY from giql.resolver import OperatorResolution from giql.resolver import ResolutionError @@ -68,6 +72,7 @@ "ExpansionContext", "OperatorExpander", "ExpanderRegistry", + "RegistrySnapshot", "REGISTRY", "register", "expand_operators", @@ -191,6 +196,25 @@ def _as_callable(expander: OperatorExpander | ExpanderFn) -> ExpanderFn: ) +@dataclass(frozen=True) +class RegistrySnapshot: + """An opaque save/restore token for :class:`ExpanderRegistry` state. + + Bundles the ``(target, operator)`` expander map and the declared-target map + captured by :meth:`ExpanderRegistry.snapshot`. Hand it back to + :meth:`ExpanderRegistry.restore` to return the registry to that state, or + compare two snapshots for equality to assert the registry returned to a + baseline (the leak-guard use). + + A snapshot is compared by equality and handed to :meth:`restore` only; it is + **not** hashable (its two ``dict`` fields are mutable), so ``hash(snapshot)`` + raises — never use one as a set member or dict key. + """ + + expanders: dict[tuple[Target, type], ExpanderFn] + targets: dict[str, Target] + + class ExpanderRegistry: """Maps ``(target, operator_type)`` to the expander that handles it. @@ -201,7 +225,10 @@ class ExpanderRegistry: 1. ``(target, op)`` — an expander registered for the *exact* active target; 2. ``(generic, op)`` — the portable expander registered for :class:`~giql.targets.GenericTarget`; - 3. ``None`` — no expander, so the caller keeps the legacy ``*_sql`` emitter. + 3. ``None`` — no expander resolves. Every built-in operator registers a + ``(generic, op)`` expander, so this only arises for a cleared registry or + an operator with no generic fallback; the pass raises rather than emitting + (there is no legacy ``*_sql`` fallback). The registry keys on the frozen, value-equal :class:`~giql.targets.Target` *instance* (two ``DuckDBTarget()`` compare and hash alike), so registering @@ -212,6 +239,7 @@ class ExpanderRegistry: def __init__(self) -> None: self._expanders: dict[tuple[Target, type], ExpanderFn] = {} + self._targets_by_name: dict[str, Target] = {} def register( self, @@ -235,30 +263,72 @@ def register( Notes ----- - A *non-generic* ``(target, operator)`` entry is intended to also act as a - join-rewrite override for operators with a built-in whole-query join - rewrite (notably :class:`~giql.expressions.Intersects`, whose binned - equi-join / DuckDB IEJoin transformers run before expansion), letting a - per-target expander assume responsibility for that rewrite. That bypass - is intended for a future INTERSECTS consumer and is **not wired by any - caller yet** — no transformer consults :meth:`has_override` here (see - #141). + Registering an expander also declares *target* by name (see + :meth:`register_target`), so a custom target becomes selectable via + ``transpile(dialect=target.name)`` as soon as any expander is registered + against it — a user overriding one operator need not also declare the + target separately. + + A *non-generic* ``(target, operator)`` entry additionally acts as a + *join-rewrite override* (:meth:`has_override`) for operators with a + built-in whole-query join rewrite (notably + :class:`~giql.expressions.Intersects`, whose binned equi-join / DuckDB + IEJoin transformers run before expansion), letting a per-target expander + assume responsibility for that rewrite — :func:`giql.transpile.transpile` + consults :meth:`has_override` and bypasses the built-in transformers when + it holds. """ self._expanders[(target, operator)] = _as_callable(expander) + self.register_target(target) + + def register_target(self, target: Target) -> None: + """Declare *target* so ``transpile(dialect=target.name)`` resolves it. + + The registry doubles as the target plugin hub: a custom + :class:`~giql.targets.Target` registered here (directly, or as a side + effect of :meth:`register`) is resolvable by name through + :func:`giql.targets.resolve_target`. This is the path for a + *capability-only* target — one that overrides no operators (so it never + calls :meth:`register`) but declares a distinct + :class:`~giql.targets.Capabilities` set / ``sqlglot_dialect``. + + The built-in :class:`~giql.targets.GenericTarget` is intentionally **not** + declared: it is the portable fallback, selected only via ``dialect=None`` + (see :func:`giql.targets.resolve_target`), never by name. The skip is + *by value* (``target == GenericTarget()``), so registering an expander + against ``GenericTarget`` is a no-op here — but a *custom* + :class:`~giql.targets.Target` subclass is declared under its own ``name`` + even if that name happens to be ``"generic"`` (it does not compare equal to + the built-in). A later registration for the same name replaces an earlier + one. + """ + if target == GenericTarget(): + return + self._targets_by_name[target.name] = target + + def target(self, name: str) -> Target | None: + """Return the registered custom target named *name*, or ``None``. + + Consulted by :func:`giql.targets.resolve_target` for any dialect name + that is not a built-in, so a registered custom target is selectable via + ``transpile(dialect=name)``. + """ + return self._targets_by_name.get(name) def resolve(self, target: Target, operator: type) -> ExpanderFn | None: """Return the expander for ``(target, operator)`` via the fallback chain. Tries the exact ``(target, op)`` entry, then the - ``(GenericTarget(), op)`` fallback, then ``None`` (legacy emitter). + ``(GenericTarget(), op)`` fallback, then ``None``. ``None`` means no + expander is registered; the :class:`ExpandOperators` pass treats that as + an internal invariant violation and raises (there is no legacy emitter). - A non-generic exact ``(target, op)`` entry is also intended to act as a + A non-generic exact ``(target, op)`` entry additionally acts as a *join-rewrite override* for operators with a built-in whole-query join - rewrite (notably :class:`~giql.expressions.Intersects`). That override is - intended for a future INTERSECTS consumer and is **not wired by any - caller yet** — resolution does not itself bypass the built-in - binned / IEJoin transformers (see :meth:`register`, :meth:`has_override`, - and #141). + rewrite (notably :class:`~giql.expressions.Intersects`); resolution itself + does not bypass the built-in binned / IEJoin transformers — + :func:`giql.transpile.transpile` does, gated on :meth:`has_override` (see + :meth:`register` and #141). """ fn = self._expanders.get((target, operator)) if fn is not None: @@ -277,12 +347,13 @@ def has_override(self, target: Target, operator: type) -> bool: ``(GenericTarget(), operator)`` fallback is *not* an override and does not count here. - Such an entry is intended to mark a target-specific override that - supersedes built-in handling (e.g. taking responsibility for the - whole-query join rewrite the built-in transformers would otherwise - perform). That mechanism is intended for a future INTERSECTS consumer and - is **not wired by any caller yet** — no transformer consults this method - in the current pipeline (see #141). + Such an entry marks a target-specific override that supersedes built-in + handling (e.g. taking responsibility for the whole-query join rewrite the + built-in transformers would otherwise perform). + :func:`giql.transpile.transpile` consults this method for + ``(target, Intersects)`` and, when it holds, bypasses the built-in + binned / IEJoin join transformers so the ``Intersects`` node flows to the + registered expander instead (see #141). """ return target != GenericTarget() and (target, operator) in self._expanders @@ -298,38 +369,44 @@ def unregister(self, target: Target, operator: type) -> None: self._expanders.pop((target, operator), None) def clear(self) -> None: - """Drop every registration. + """Drop every registration — both expanders and declared targets. The bulk form of :meth:`unregister`, and the **public reset** for the process-wide :data:`REGISTRY` — e.g. a test fixture that saves and - restores the registry around a body that registers custom expanders. + restores the registry around a body that registers custom expanders or + targets. """ self._expanders.clear() + self._targets_by_name.clear() - def snapshot(self) -> dict[tuple[Target, type], ExpanderFn]: - """Return a shallow copy of the current registrations. + def snapshot(self) -> RegistrySnapshot: + """Return a shallow copy of the current registrations and targets. The save half of a save/restore seam that supports test baseline-isolation: capture the baseline with this and hand it back to - :meth:`restore` afterward, so the built-in expanders registered at import - survive an isolating fixture that would otherwise :meth:`clear` them - permanently. + :meth:`restore` afterward, so the built-in registrations survive an + isolating fixture that would otherwise :meth:`clear` them permanently. - The returned dict is a fresh mapping (mutating it does not affect the - registry), keyed by the same ``(target, operator)`` tuples. + The returned :class:`RegistrySnapshot` is a fresh, opaque value bundling + both the ``(target, operator)`` expander map and the declared-target map; + mutating the registry afterward does not affect it, and two snapshots + compare equal when the registry is in the same state. """ - return dict(self._expanders) + return RegistrySnapshot(dict(self._expanders), dict(self._targets_by_name)) - def restore(self, snapshot: dict[tuple[Target, type], ExpanderFn]) -> None: + def restore(self, snapshot: RegistrySnapshot) -> None: """Replace all registrations with those captured by :meth:`snapshot`. The restore half of the save/restore seam that supports test baseline-isolation. Drops every current entry and re-installs exactly the - *snapshot* contents, so a fixture can return the registry to a previously - captured baseline regardless of what its body registered or cleared. + *snapshot* contents (expanders and declared targets), so a fixture can + return the registry to a previously captured baseline regardless of what + its body registered or cleared. """ self._expanders.clear() - self._expanders.update(snapshot) + self._expanders.update(snapshot.expanders) + self._targets_by_name.clear() + self._targets_by_name.update(snapshot.targets) def __contains__(self, key: tuple[Target, type]) -> bool: """Whether an *exact* ``(target, operator)`` entry is registered. @@ -359,8 +436,8 @@ def __bool__(self) -> bool: #: The process-wide registry the :func:`register` decorator writes to and the #: :class:`ExpandOperators` pass reads from. The built-in expanders register into -#: it at import time via :mod:`giql.expanders`; the pass rewrites a node only when -#: an expander resolves here (and the operator is flagged ``GIQL_EXPAND``). +#: it at import time via :mod:`giql.expanders`; the pass rewrites every GIQL +#: operator node by resolving its expander here. REGISTRY = ExpanderRegistry() @@ -414,34 +491,21 @@ def decorator( return decorator -# The GIQL operator expression classes the pass inspects. Membership alone does -# not opt an operator in: the per-class ``GIQL_EXPAND`` flag plus a registered -# expander do (see module docstring). Imported lazily as a precaution; there is no -# real cycle (``canonicalizer.py`` imports the same classes eagerly at module -# level), so a later step may hoist these to module scope to match that sibling. -def _giql_operators() -> tuple[type, ...]: - """Return the GIQL operator classes, imported lazily as a precaution.""" - from giql.expressions import Contains - from giql.expressions import GIQLCluster - from giql.expressions import GIQLDisjoin - from giql.expressions import GIQLDistance - from giql.expressions import GIQLMerge - from giql.expressions import GIQLNearest - from giql.expressions import Intersects - from giql.expressions import SpatialSetPredicate - from giql.expressions import Within - - return ( - GIQLDisjoin, - GIQLNearest, - GIQLDistance, - GIQLCluster, - GIQLMerge, - Intersects, - Contains, - Within, - SpatialSetPredicate, - ) +# The GIQL operator expression classes the pass inspects. Every one is migrated, +# so each resolves an expander through the registry (see module docstring). +# Imported eagerly at module scope, as ``canonicalizer.py`` imports the same +# classes: ``giql.expressions`` does not import this module, so there is no cycle. +_GIQL_OPERATORS: tuple[type, ...] = ( + GIQLDisjoin, + GIQLNearest, + GIQLDistance, + GIQLCluster, + GIQLMerge, + Intersects, + Contains, + Within, + SpatialSetPredicate, +) def expand_operators( @@ -450,19 +514,17 @@ def expand_operators( tables: Tables, registry: ExpanderRegistry | None = None, ) -> exp.Expression: - """Replace each opted-in GIQL operator with its registry expansion. + """Replace every GIQL operator node with its registry expansion. Pass 3 of the normalization pipeline (epic #137). Runs after - :func:`giql.canonicalizer.canonicalize_coordinates`. For every GIQL operator - node it dispatches to the new AST-expansion path **only when** the operator's - class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for - ``(target, operator type)`` through its fallback chain; otherwise the node is - left untouched and the legacy ``*_sql`` emitter handles it. + :func:`giql.canonicalizer.canonicalize_coordinates`. Every GIQL operator is + migrated, so the pass dispatches *every* operator node: it resolves an expander + for ``(target, operator type)`` through the registry's fallback chain and + replaces the node with the AST that expander returns. A resolution miss is an + internal invariant violation (a built-in operator always has at least a + ``(generic, op)`` expander) and raises — there is no legacy ``*_sql`` fallback. - The pass mutates and returns *expression* in place. It touches only nodes - whose operator is flagged ``GIQL_EXPAND`` and resolves an expander; for every - other operator it is a no-op, leaving the emitted SQL byte-identical, so the - existing suite is the migration oracle. + The pass mutates and returns *expression* in place. Parameters ---------- @@ -479,12 +541,11 @@ class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for Returns ------- exp.Expression - The same *expression*, with each opted-in operator node that resolves an - expander replaced by its target-specific expansion; nodes that are - unflagged or resolve no expander are left untouched. + The same *expression*, with each operator node replaced by its + target-specific expansion. """ reg = registry if registry is not None else REGISTRY - operators = _giql_operators() + operators = _GIQL_OPERATORS alias_seq = name_sequence(EXPAND_ALIAS_PREFIX) # Collect first, then mutate: replacing nodes mid-walk is unsafe. @@ -492,11 +553,19 @@ class sets ``GIQL_EXPAND = True`` *and* the registry resolves an expander for for node in expression.walk(): if not isinstance(node, operators): continue - if not getattr(node, "GIQL_EXPAND", False): - continue fn = reg.resolve(target, type(node)) if fn is None: - continue + # Every built-in operator registers at least a ``(generic, op)`` + # expander at import time, so a miss means the registry was cleared or + # a custom target shadowed an operator without providing one — an + # internal/config error, not user input. Fail loudly rather than + # leaving an un-serializable GIQL node for the stock serializer (there + # is no legacy ``*_sql`` fallback anymore). + raise ValueError( + f"No expander registered for {type(node).__name__} on target " + f"{target.name!r}; the ExpandOperators pass cannot leave a GIQL " + "operator node un-expanded." + ) pending.append((node, fn)) # Replace deepest-first: replacing an ancestor node detaches any collected diff --git a/src/giql/expanders/_distance.py b/src/giql/expanders/_distance.py new file mode 100644 index 0000000..eb28f9f --- /dev/null +++ b/src/giql/expanders/_distance.py @@ -0,0 +1,117 @@ +"""Shared distance-CASE string builder for the NEAREST expander. + +The private ``_``-prefix keeps this module out of the ``expanders`` package +auto-discovery (see :mod:`giql.expanders`), so it is a plain helper module rather +than a registered expander. + +.. note:: + + KEEP IN SYNC: :func:`generate_distance_case` here and the AST builder in + :mod:`giql.expanders.distance` (``expand_distance``) produce the *same* + distance CASE by two routes. DISTANCE migrated to the AST expander (epic + #137, issue #140); this string form survives because NEAREST assembles its + ORDER BY / filter math as SQL fragments (string-first) and splices the CASE + into them. Any change to the distance math here must be mirrored in the + expander (and vice versa); the parity test in ``tests/test_distance_udf.py`` + guards drift. +""" + +from __future__ import annotations + + +def generate_distance_case( + chrom_a: str, + start_a: str, + end_a: str, + strand_a: str | None, + chrom_b: str, + start_b: str, + end_b: str, + strand_b: str | None, + stranded: bool = False, + signed: bool = False, +) -> str: + """Generate a SQL CASE expression for distance calculation. + + Distances follow bedtools ``closest -d`` semantics: overlapping + intervals report ``0``, book-ended (adjacent) intervals where + ``A.end == B.start`` in half-open coordinates report ``1``, and a raw + half-open gap of N bases reports ``N + 1``. The ``+ 1`` is applied to + the absolute gap magnitude before any directional sign, so a downstream + book-ended pair reports ``+1`` and an upstream one ``-1`` in signed mode. + + :param chrom_a: + Chromosome column for interval A + :param start_a: + Start column for interval A + :param end_a: + End column for interval A + :param strand_a: + Strand column for interval A (None if not stranded) + :param chrom_b: + Chromosome column for interval B + :param start_b: + Start column for interval B + :param end_b: + End column for interval B + :param strand_b: + Strand column for interval B (None if not stranded) + :param stranded: + Whether to use strand-aware distance calculation + :param signed: + Whether to return signed distance (negative for upstream, positive for + downstream) + :return: + SQL CASE expression + """ + if not stranded or strand_a is None or strand_b is None: + # Basic distance calculation without strand awareness + if signed: + # Signed distance: negative for upstream (B before A), + # positive for downstream (B after A) + return ( + f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " + f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " + f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " + f"ELSE -({start_a} - {end_b} + 1) END" + ) + # Unsigned (absolute) distance + return ( + f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " + f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " + f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " + f"ELSE ({start_a} - {end_b} + 1) END" + ) + + # Stranded distance calculation + # Return NULL if either strand is '.', '?', or NULL + # Calculate distance and multiply by -1 if first interval is on '-' strand + if signed: + # Stranded + signed: apply strand flip AND directional sign + return ( + f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " + f"WHEN {strand_a} IS NULL OR {strand_b} IS NULL THEN NULL " + f"WHEN {strand_a} = '.' OR {strand_a} = '?' THEN NULL " + f"WHEN {strand_b} = '.' OR {strand_b} = '?' THEN NULL " + f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " + f"WHEN {end_a} <= {start_b} THEN " + f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a} + 1) " + f"ELSE ({start_b} - {end_a} + 1) END " + f"ELSE " + f"CASE WHEN {strand_a} = '-' THEN ({start_a} - {end_b} + 1) " + f"ELSE -({start_a} - {end_b} + 1) END END" + ) + # Stranded but not signed: apply strand flip only + return ( + f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " + f"WHEN {strand_a} IS NULL OR {strand_b} IS NULL THEN NULL " + f"WHEN {strand_a} = '.' OR {strand_a} = '?' THEN NULL " + f"WHEN {strand_b} = '.' OR {strand_b} = '?' THEN NULL " + f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " + f"WHEN {end_a} <= {start_b} THEN " + f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a} + 1) " + f"ELSE ({start_b} - {end_a} + 1) END " + f"ELSE " + f"CASE WHEN {strand_a} = '-' THEN -({start_a} - {end_b} + 1) " + f"ELSE ({start_a} - {end_b} + 1) END END" + ) diff --git a/src/giql/expanders/_params.py b/src/giql/expanders/_params.py new file mode 100644 index 0000000..60eaa83 --- /dev/null +++ b/src/giql/expanders/_params.py @@ -0,0 +1,37 @@ +"""Shared operator-parameter coercions for the expanders. + +The private ``_``-prefix keeps this module out of the ``expanders`` package +auto-discovery (see :mod:`giql.expanders`), so it is a plain helper module rather +than a registered expander. +""" + +from __future__ import annotations + +from sqlglot import exp + + +def coerce_bool_param(param: exp.Expression | None) -> bool: + """Coerce an optional operator boolean argument to a Python ``bool``. + + The ``stranded`` / ``signed`` keyword arguments of DISTANCE and NEAREST are + read the same way: an absent argument is ``False``, an + :class:`sqlglot.exp.Boolean` node yields ``bool(node.this)``, and anything + else is truthy when its rendered text is ``TRUE`` / ``1`` / ``YES``. The + :func:`bool` coercion guarantees a real ``bool`` regardless of what the + parser stored on the node. + + Parameters + ---------- + param : exp.Expression | None + The ``stranded`` or ``signed`` argument node, or ``None`` if absent. + + Returns + ------- + bool + The coerced Python boolean (``False`` when the argument is absent). + """ + if not param: + return False + if isinstance(param, exp.Boolean): + return bool(param.this) + return str(param).upper() in ("TRUE", "1", "YES") diff --git a/src/giql/expanders/disjoin.py b/src/giql/expanders/disjoin.py index f784805..f0c178c 100644 --- a/src/giql/expanders/disjoin.py +++ b/src/giql/expanders/disjoin.py @@ -8,7 +8,7 @@ set. This module is the AST-expansion replacement for the legacy -``BaseGIQLGenerator.giqldisjoin_sql`` emitter. It assembles the same WITH-CTE +``giqldisjoin_sql`` generator emitter. It assembles the same WITH-CTE subquery, parses it back into a sqlglot :class:`~sqlglot.expressions.Expression`, and returns that node so the active target's serializer renders it — dissolving the emit-time string special-case left by epic #114. @@ -102,7 +102,7 @@ def expand_disjoin(node: GIQLDisjoin, ctx: ExpansionContext) -> exp.Expression: def _build_disjoin_sql(node: GIQLDisjoin, ctx: ExpansionContext) -> str: """Assemble the DISJOIN WITH-CTE subquery SQL string for the active target. - Mirrors the legacy ``BaseGIQLGenerator.giqldisjoin_sql`` shape one named + Mirrors the legacy ``giqldisjoin_sql`` emitter's shape one named ``__giql_dj_*`` CTE at a time, with the #153 alias fix and the capability-driven passthrough applied. """ @@ -317,9 +317,9 @@ def _disjoin_resolution( ref = resolution.slot("reference") if ref is not None: if ref.kind == "subquery": - # Serializing the subquery with sqlglot's stock generator (not - # BaseGIQLGenerator) is safe because nested GIQL operators expand - # deepest-first: any GIQL construct inside this operand is already + # Serializing the subquery with sqlglot's stock generator is safe + # because nested GIQL operators expand deepest-first: any GIQL + # construct inside this operand is already # rewritten to standard AST by the time this outer DISJOIN expands, # so the stock generator only ever round-trips plain SQL here. ref_sql = reference.sql(dialect=GIQLDialect) diff --git a/src/giql/expanders/distance.py b/src/giql/expanders/distance.py index acd2c81..dd6f6c0 100644 --- a/src/giql/expanders/distance.py +++ b/src/giql/expanders/distance.py @@ -8,8 +8,8 @@ every target. DISTANCE emits identical SQL on DuckDB, DataFusion, and the generic baseline, so no per-target override is needed. -The CASE this expander builds matches -:meth:`giql.generators.base.BaseGIQLGenerator._generate_distance_case` exactly +The CASE this expander builds matches the string builder +:func:`giql.expanders._distance.generate_distance_case` exactly (bedtools ``closest -d`` semantics): overlapping intervals report ``0``, book-ended (adjacent) intervals report ``1``, and a raw half-open gap of ``N`` bases reports ``N + 1``. The ``+ 1`` is applied to the absolute gap magnitude @@ -32,6 +32,7 @@ from giql.dialect import GIQLDialect from giql.expander import ExpansionContext from giql.expander import register +from giql.expanders._params import coerce_bool_param from giql.expressions import GIQLDistance from giql.resolver import ResolvedColumn from giql.targets import GenericTarget @@ -83,36 +84,6 @@ def _gap(minuend: str, subtrahend: str) -> exp.Expression: return exp.paren(exp.Add(this=diff, expression=exp.Literal.number(1))) -def _bool_param(param: exp.Expression | None) -> bool: - """Coerce an optional DISTANCE boolean argument to a Python ``bool``. - - Mirrors ``BaseGIQLGenerator._extract_bool_param`` in how it reads the - ``stranded`` / ``signed`` keyword arguments, with one *intentional* - divergence: for an :class:`sqlglot.exp.Boolean` node this returns - ``bool(param.this)`` (a true Python ``bool``), whereas the legacy - ``_extract_bool_param`` returns the raw ``param_expr.this`` (already a - ``bool`` in practice, but unhardened). The coercion is strictly safer — it - guarantees a ``bool`` regardless of what the parser stored — and never - changes the observed branch selection, so the two stay behaviorally - equivalent. See TODO(#146): folding these two readers together. - - Parameters - ---------- - param : exp.Expression | None - The ``stranded`` or ``signed`` argument node, or ``None`` if absent. - - Returns - ------- - bool - The coerced Python boolean (``False`` when the argument is absent). - """ - if not param: - return False - if isinstance(param, exp.Boolean): - return bool(param.this) - return str(param).upper() in ("TRUE", "1", "YES") - - def _operand(ctx: ExpansionContext, arg: str, position: str) -> ResolvedColumn: """Return the resolved column for one DISTANCE interval operand. @@ -141,7 +112,7 @@ def _operand(ctx: ExpansionContext, arg: str, position: str) -> ResolvedColumn: ValueError If the operand was deferred (a literal range or unresolved column). """ - # TODO(#146): this read-required-column-or-raise pattern is duplicated across + # TODO: this read-required-column-or-raise pattern is duplicated across # expanders; hoist it to a shared ``ExpansionContext.require_column`` helper # once a second expander needs it. resolution = ctx.resolution @@ -316,13 +287,12 @@ def _prepend_strand_guards( return case -# KEEP IN SYNC: this expander and -# ``BaseGIQLGenerator._generate_distance_case`` (base.py) build the *same* -# distance CASE by two routes. The legacy method is retained only because -# NEAREST still calls it for its ORDER BY / filter math; once NEAREST migrates -# to the expander path that method can be deleted and this duplication retired. -# Until then, any change to the distance math here must be mirrored there (and -# vice versa). The parity test in tests/test_distance_udf.py guards the drift. +# KEEP IN SYNC: this expander (AST) and the string builder +# ``giql.expanders._distance.generate_distance_case`` build the *same* distance +# CASE by two routes. The string form is retained because NEAREST assembles its +# ORDER BY / filter math string-first and splices the CASE in. Any change to the +# distance math here must be mirrored there (and vice versa). The parity test in +# tests/test_distance_udf.py guards the drift. @register(GenericTarget, GIQLDistance) def expand_distance(node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: """Expand a ``GIQLDistance`` node into a standard SQL ``CASE`` expression. @@ -345,8 +315,8 @@ def expand_distance(node: exp.Expression, ctx: ExpansionContext) -> exp.Expressi The distance ``CASE`` that replaces the operator node and is rendered by the active target's serializer. """ - stranded = _bool_param(node.args.get("stranded")) - signed = _bool_param(node.args.get("signed")) + stranded = coerce_bool_param(node.args.get("stranded")) + signed = coerce_bool_param(node.args.get("signed")) col_a = _operand(ctx, "this", "first") col_b = _operand(ctx, "expression", "second") diff --git a/src/giql/expanders/intersects.py b/src/giql/expanders/intersects.py index cbda7d6..59b7b7a 100644 --- a/src/giql/expanders/intersects.py +++ b/src/giql/expanders/intersects.py @@ -1,8 +1,8 @@ """Generic expanders for the spatial predicates and set predicates (epic #137). Migrates INTERSECTS / CONTAINS / WITHIN and the ``ANY`` / ``ALL`` set predicates -off the legacy ``*_sql`` emitters on :class:`giql.generators.base.BaseGIQLGenerator` -and onto the operator-expander registry. Each expander turns one predicate node +off the legacy ``*_sql`` generator emitters and onto the operator-expander +registry. Each expander turns one predicate node into standard sqlglot AST built from the pass-1 :class:`~giql.resolver.ResolvedColumn` metadata (already canonicalized to 0-based half-open by pass 2), so the emitted SQL is byte-identical to the strings the legacy emitter produced. @@ -58,7 +58,7 @@ def _fragment(fragment: str) -> exp.Expression: def _predicate_column(ctx: ExpansionContext, arg: str) -> ResolvedColumn: """Return the :class:`ResolvedColumn` for predicate operand *arg*. - Mirrors :meth:`giql.generators.base.BaseGIQLGenerator._predicate_operand`: the + Mirrors the legacy ``_predicate_operand`` emitter helper: the expander consumes only the pass-1 resolution; a missing column means pass 1 did not run (an internal invariant violation), so raise the historical message. """ @@ -78,7 +78,7 @@ def _range_predicate( ) -> exp.Expression: """Build the boolean AST for ``column ``. - Reproduces :meth:`BaseGIQLGenerator._generate_range_predicate` as AST. The + Reproduces the legacy ``_generate_range_predicate`` emitter as AST. The column fragments are already canonical 0-based half-open (pass 2); the parsed range is canonicalized by the caller. Returns a parenthesized boolean. """ @@ -129,7 +129,7 @@ def _column_join( ) -> exp.Expression: """Build the boolean AST for a column-to-column spatial predicate. - Reproduces :meth:`BaseGIQLGenerator._generate_column_join` as AST. Both + Reproduces the legacy ``_generate_column_join`` emitter as AST. Both operands' fragments are pre-canonicalized (pass 2). Returns a parenthesized boolean. """ @@ -218,7 +218,7 @@ def expand_spatial_set( ) -> exp.Expression: """Expand a quantified set predicate (``ANY`` / ``ALL``) to boolean SQL AST. - Reproduces :meth:`BaseGIQLGenerator._generate_spatial_set`: the single left + Reproduces the legacy ``_generate_spatial_set`` emitter: the single left column is compared against every literal range, and the per-range conditions are OR-combined for ``ANY`` / AND-combined for ``ALL``, all wrapped in one outer paren. diff --git a/src/giql/expanders/nearest.py b/src/giql/expanders/nearest.py index c2144e8..8c8e329 100644 --- a/src/giql/expanders/nearest.py +++ b/src/giql/expanders/nearest.py @@ -24,11 +24,11 @@ every target — DataFusion included — uses the LATERAL/standalone form unchanged; only the *correlated* shape needs the fallback. -The expander reuses :class:`giql.generators.base.BaseGIQLGenerator`'s -``_generate_distance_case`` (shared with DISTANCE, #140) and ``_nearest_*`` -passthrough/diagnostic helpers — all static, so they are called on the class with -no generator instance — then parses the assembled SQL fragments into AST so the -emitted SQL is reserialized by the active target's serializer. +The expander assembles its distance/passthrough/filter SQL as string fragments +(the ``_nearest_*`` helpers below and +:func:`giql.expanders._distance.generate_distance_case`, shared in spirit with +DISTANCE, #140), then parses the assembled fragments into AST so the emitted SQL +is reserialized by the active target's serializer. """ from __future__ import annotations @@ -36,14 +36,21 @@ from sqlglot import exp from sqlglot import parse_one +from giql.canonical import decanonical_end +from giql.canonical import decanonical_start from giql.dialect import GIQLDialect from giql.expander import EXPAND_ALIAS_PREFIX from giql.expander import ExpansionContext from giql.expander import register +from giql.expanders._distance import generate_distance_case +from giql.expanders._params import coerce_bool_param from giql.expressions import GIQLNearest -from giql.generators.base import BaseGIQLGenerator +from giql.range_parser import RangeParser +from giql.resolver import META_KEY +from giql.resolver import OperatorResolution from giql.resolver import ResolvedInterval from giql.resolver import ResolvedRef +from giql.table import Table from giql.targets import Capabilities from giql.targets import GenericTarget @@ -56,6 +63,146 @@ _REF_KEY_PREFIX = f"{EXPAND_ALIAS_PREFIX}rk_" +def _nearest_output_encoding( + expression: GIQLNearest, target_ref: ResolvedRef +) -> Table | None: + """Return the target's declared encoding for NEAREST's row passthrough. + + ``CanonicalizeCoordinates`` (pass 2) records the original + :class:`~giql.table.Table` on the resolution when it wraps a non-canonical + target in a ``__giql_canon_*`` CTE (blanking the slot's own ``table``). For + an unwrapped target — a canonical registered table, or any target when the + pass did not run — the slot's own ``table`` carries the (identity) encoding. + + :param expression: + GIQLNearest expression node + :param target_ref: + The resolved target reference (post pass 2) + :return: + The target's declared :class:`~giql.table.Table`, or ``None`` + """ + resolution = expression.meta.get(META_KEY) + if isinstance(resolution, OperatorResolution): + preserved = resolution.output_tables.get("this") + if preserved is not None: + return preserved + return target_ref.table + + +def _nearest_passthrough( + table_name: str, + target_start: str, + target_end: str, + output_table: Table | None, + capabilities: Capabilities, +) -> str: + """Project the target's full row, de-canonicalizing the interval columns. + + NEAREST passes the whole target row through (``SELECT {table_name}.*``) + alongside the synthesized, encoding-invariant ``distance`` column. When the + target's declared encoding is canonical 0-based half-open the row passes + through as a plain ``{table_name}.*`` — the byte-identical identity fast + path. When it is non-canonical the interval columns, canonical inside the + ``__giql_canon_*`` CTE the target was rewritten to, are de-canonicalized + back into that encoding so the passed-through interval matches the target's + own convention. + + The emit strategy is chosen from *capabilities*, following the precedent of + :func:`giql.expanders.disjoin._disjoin_passthrough` (issue #145) — the same + two emit forms: + + * ``{table_name}.* REPLACE (...)`` when ``supports_star_replace`` holds — + substitutes start/end in place; + * the portable ``{table_name}.* EXCEPT (start, end), , `` form + otherwise (the generic baseline / DataFusion family). Row-equivalent but + not column-order-equivalent, and not DuckDB-runnable. + + :param table_name: + The relation the row is selected from (the canon CTE name when wrapped, + else the registered table name) — also the column qualifier. + :param target_start: + Physical start column name + :param target_end: + Physical end column name + :param output_table: + The target's declared :class:`~giql.table.Table`, or ``None`` + :param capabilities: + The active target's :class:`~giql.targets.Capabilities`. + :return: + The passthrough projection fragment + """ + if output_table is None or ( + output_table.coordinate_system == "0based" + and output_table.interval_type == "half_open" + ): + return f"{table_name}.*" + pt_start = decanonical_start(f'{table_name}."{target_start}"', output_table) + pt_end = decanonical_end(f'{table_name}."{target_end}"', output_table) + if capabilities.supports_star_replace: + return ( + f"{table_name}.* REPLACE " + f'({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' + ) + # Portable form for engines without ``* REPLACE`` (generic / DataFusion): + # drop the interval columns from the star and re-project them recomputed. + return ( + f'{table_name}.* EXCEPT ("{target_start}", "{target_end}"), ' + f'{pt_start} AS "{target_start}", {pt_end} AS "{target_end}"' + ) + + +def _raise_nearest_reference_error( + expression: GIQLNearest, + resolution: OperatorResolution | None, +) -> None: + """Raise the historical diagnostic for an unresolved NEAREST reference. + + ResolveOperatorRefs (pass 1) defers a reference slot it cannot resolve; this + re-raises the historical pre-pass error verbatim. The implicit-outer failures + rely on the :class:`~giql.resolver.SlotDeferral` the pass records (the + ancestor walk that distinguished them now lives in the pass); a literal-range + parse failure is reproduced by re-parsing. + + :param expression: + GIQLNearest expression node + :param resolution: + The attached resolution metadata, if any + :raises ValueError: + Always — with the matching historical message + """ + reference = expression.args.get("reference") + + if reference is None: + # An absent reference is a correlated (implicit-outer) placement that the + # resolver could not tie to a registered outer table; consult the + # recorded deferral for the specific historical message. + deferral = ( + resolution.deferral("reference") if resolution is not None else None + ) + if deferral is not None and deferral.reason == "implicit_outer_unregistered": + raise ValueError( + f"Outer table '{deferral.detail}' not found in tables. " + "Please specify reference parameter explicitly." + ) + raise ValueError( + "Could not find outer table in LATERAL join context. " + "Please specify reference parameter explicitly." + ) + + # An explicit reference that deferred is a literal range that failed to + # parse (column references always resolve). Re-parse to surface the + # original parse error in the historical message. + reference_sql = reference.sql(dialect=GIQLDialect) + range_str = reference_sql.strip("'\"") + try: + RangeParser.parse(range_str).to_zero_based_half_open() + except Exception as e: + raise ValueError( + f"Could not parse reference genomic range: {range_str}. Error: {e}" + ) + raise ValueError(f"Could not parse reference genomic range: {range_str}.") + + def _nearest_params( expression: GIQLNearest, ) -> tuple[int, int | None, bool, bool]: @@ -66,8 +213,8 @@ def _nearest_params( max_distance = expression.args.get("max_distance") max_dist_value = int(str(max_distance)) if max_distance else None - is_stranded = BaseGIQLGenerator._extract_bool_param(expression.args.get("stranded")) - is_signed = BaseGIQLGenerator._extract_bool_param(expression.args.get("signed")) + is_stranded = coerce_bool_param(expression.args.get("stranded")) + is_signed = coerce_bool_param(expression.args.get("signed")) return k_value, max_dist_value, is_stranded, is_signed @@ -89,9 +236,9 @@ def _distance_and_filters( ORDER BY tiebreaker from the target columns itself. ``capabilities`` is the active target's :class:`~giql.targets.Capabilities`, - forwarded to :meth:`BaseGIQLGenerator._nearest_passthrough` to choose the - target's de-canonicalization emit form (``* REPLACE`` vs the portable - ``* EXCEPT``); both call sites pass ``ctx.capabilities``. + forwarded to :func:`_nearest_passthrough` to choose the target's + de-canonicalization emit form (``* REPLACE`` vs the portable ``* EXCEPT``); + both call sites pass ``ctx.capabilities``. ``ref_fragments`` optionally overrides the reference ``(chrom, start, end, strand)`` SQL fragments. The LATERAL form consumes the resolution's @@ -103,8 +250,8 @@ def _distance_and_filters( target_chrom, target_start, target_end = target_ref.cols _k_value, max_dist_value, is_stranded, is_signed = _nearest_params(expression) - output_table = BaseGIQLGenerator._nearest_output_encoding(expression, target_ref) - passthrough = BaseGIQLGenerator._nearest_passthrough( + output_table = _nearest_output_encoding(expression, target_ref) + passthrough = _nearest_passthrough( table_name, target_start, target_end, output_table, capabilities ) @@ -129,7 +276,7 @@ def _distance_and_filters( target_start_expr = f'{table_name}."{target_start}"' target_end_expr = f'{table_name}."{target_end}"' - distance_expr = BaseGIQLGenerator._generate_distance_case( + distance_expr = generate_distance_case( ref_chrom, ref_start, ref_end, @@ -466,8 +613,7 @@ def expand_nearest(node: exp.Expression, ctx: ExpansionContext) -> exp.Expressio ref = resolution.slot("reference") if not isinstance(ref, ResolvedInterval): - mode = BaseGIQLGenerator._detect_nearest_mode(node) - BaseGIQLGenerator._raise_nearest_reference_error(node, mode, resolution) + _raise_nearest_reference_error(node, resolution) # A literal-range reference is uncorrelated even under CROSS JOIN LATERAL: its # endpoints are constants, not outer-row columns, so the subquery stands alone diff --git a/src/giql/expressions.py b/src/giql/expressions.py index ffbf017..900b69e 100644 --- a/src/giql/expressions.py +++ b/src/giql/expressions.py @@ -118,12 +118,6 @@ class Intersects(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE - #: Migrated to the ExpandOperators registry (#141). A literal-range or - #: residual column-to-column INTERSECTS *predicate* expands through - #: ``giql.expanders.intersects``; a column-to-column INTERSECTS *join* is - #: consumed by the capability-gated binned / IEJoin pre-pass transformers - #: before this pass runs, so the predicate expander never sees it. - GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -138,9 +132,6 @@ class Contains(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE - #: Migrated to the ExpandOperators registry (#141); expands through - #: ``giql.expanders.intersects``. - GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -155,9 +146,6 @@ class Within(SpatialPredicate): """ GIQL_CANONICALIZE = _CANONICALIZE - #: Migrated to the ExpandOperators registry (#141); expands through - #: ``giql.expanders.intersects``. - GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -181,9 +169,6 @@ class SpatialSetPredicate(exp.Expression): } GIQL_CANONICALIZE = _CANONICALIZE - #: Migrated to the ExpandOperators registry (#141); expands through - #: ``giql.expanders.intersects``. - GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -235,14 +220,10 @@ class GIQLCluster(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } - #: Migrated to the ExpandOperators pass (epic #137, issue #144). CLUSTER is - #: expanded by ``giql.expanders.cluster`` — a whole-query rewrite into the - #: two-level ``lag_calc`` form — replacing the legacy - #: ``giql.transformer.ClusterTransformer`` pre-pass transformer. Note CLUSTER - #: deliberately does NOT set ``GIQL_CANONICALIZE``: the expander derives its - #: columns from the FROM table, so pass 2 is intentionally a no-op here and the - #: emitted SQL stays byte-identical to the legacy pre-pass output. - GIQL_EXPAND = True + # CLUSTER (expanded by ``giql.expanders.cluster`` — a whole-query rewrite into + # the two-level ``lag_calc`` form) deliberately does NOT set + # ``GIQL_CANONICALIZE``: the expander derives its columns from the FROM table, + # so pass 2 is intentionally a no-op here. @classmethod def from_arg_list(cls, args): @@ -285,14 +266,10 @@ class GIQLMerge(exp.Func): "predicate": False, # pairwise boolean gate (current row vs PREV(col)) } - #: Migrated to the ExpandOperators pass (epic #137, issue #144). MERGE is - #: expanded by ``giql.expanders.merge`` — a whole-query rewrite into the - #: clustered-aggregation form (built on CLUSTER) — replacing the legacy - #: ``giql.transformer.MergeTransformer`` pre-pass transformer. Like CLUSTER, - #: MERGE deliberately does NOT set ``GIQL_CANONICALIZE`` (columns come from the - #: FROM table), so pass 2 is intentionally a no-op and the SQL stays - #: byte-identical to the legacy pre-pass output. - GIQL_EXPAND = True + # MERGE (expanded by ``giql.expanders.merge`` — a whole-query rewrite into the + # clustered-aggregation form, built on CLUSTER) deliberately does NOT set + # ``GIQL_CANONICALIZE`` (columns come from the FROM table), so pass 2 is + # intentionally a no-op here. @classmethod def from_arg_list(cls, args): @@ -330,10 +307,6 @@ class GIQLDistance(exp.Func): } GIQL_CANONICALIZE = _CANONICALIZE - # Migrated to the registry's AST-expansion path (epic #137, issue #140): the - # generic expander in giql.expanders.distance builds the CASE; the legacy - # giqldistance_sql emitter is gone. - GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"column"}), required=True), @@ -381,11 +354,6 @@ class GIQLNearest(exp.Func): #: half-open) operands are left untouched and the emitted SQL stays #: byte-identical. GIQL_CANONICALIZE = _CANONICALIZE - #: Migrated to the ExpandOperators pass (epic #137, issue #142): NEAREST is - #: expanded by ``giql.expanders.nearest`` — the portable correlated LATERAL - #: subquery where ``supports_lateral`` holds, a decorrelated window-function - #: form otherwise. The legacy ``giqlnearest_sql`` emitter has been removed. - GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), @@ -435,11 +403,6 @@ class GIQLDisjoin(exp.Func): #: (0-based half-open) operands are left unwrapped and the emitted SQL stays #: byte-identical. GIQL_CANONICALIZE = True - #: Opt DISJOIN into the ExpandOperators pass (epic #137, issue #143). DISJOIN - #: is migrated to a registered expander (``giql.expanders.disjoin``), so the - #: pass replaces the node with the expander's AST and the legacy - #: ``giqldisjoin_sql`` emitter is removed. - GIQL_EXPAND = True GIQL_SLOTS = ( SlotSpec("this", frozenset({"registered_table"}), required=True), diff --git a/src/giql/generators/__init__.py b/src/giql/generators/__init__.py deleted file mode 100644 index ca8cb16..0000000 --- a/src/giql/generators/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""SQL generators for GIQL transpilation.""" - -from giql.generators.base import BaseGIQLGenerator - -__all__ = ["BaseGIQLGenerator"] diff --git a/src/giql/generators/base.py b/src/giql/generators/base.py deleted file mode 100644 index a33480b..0000000 --- a/src/giql/generators/base.py +++ /dev/null @@ -1,339 +0,0 @@ -from sqlglot import exp -from sqlglot.generator import Generator - -from giql.canonical import decanonical_end -from giql.canonical import decanonical_start -from giql.dialect import GIQLDialect -from giql.expressions import GIQLNearest -from giql.range_parser import RangeParser -from giql.resolver import META_KEY -from giql.resolver import OperatorResolution -from giql.resolver import ResolvedColumn -from giql.resolver import ResolvedRef -from giql.table import Table -from giql.table import Tables -from giql.targets import Capabilities - - -class BaseGIQLGenerator(Generator): - """Base generator that outputs standard SQL. - - Works with any SQL database that supports: - - - Basic comparison operators (<, >, =, AND, OR) - - String literals - - Numeric comparisons - - This generator uses only SQL-92 compatible constructs, ensuring - compatibility with virtually all SQL databases. - """ - - def __init__(self, tables: Tables | None = None, **kwargs): - super().__init__(**kwargs) - self.tables = tables or Tables() - - # INTERSECTS / CONTAINS / WITHIN and the ANY/ALL set predicates are migrated - # to the ExpandOperators registry (#141): they expand to standard boolean AST - # in ``giql.expanders.intersects`` before generation, so the generator no - # longer carries ``intersects_sql`` / ``contains_sql`` / ``within_sql`` / - # ``spatialsetpredicate_sql`` emitters or their ``_generate_spatial_*`` / - # ``_predicate_operand`` helpers. - - @staticmethod - def _nearest_output_encoding( - expression: GIQLNearest, target_ref: ResolvedRef - ) -> Table | None: - """Return the target's declared encoding for NEAREST's row passthrough. - - ``CanonicalizeCoordinates`` (pass 2) records the original - :class:`~giql.table.Table` on the resolution when it wraps a non-canonical - target in a ``__giql_canon_*`` CTE (blanking the slot's own ``table``). - For an unwrapped target — a canonical registered table, or any target when - the pass did not run — the slot's own ``table`` carries the (identity) - encoding. - - :param expression: - GIQLNearest expression node - :param target_ref: - The resolved target reference (post pass 2) - :return: - The target's declared :class:`~giql.table.Table`, or ``None`` - """ - resolution = expression.meta.get(META_KEY) - if isinstance(resolution, OperatorResolution): - preserved = resolution.output_tables.get("this") - if preserved is not None: - return preserved - return target_ref.table - - @staticmethod - def _nearest_passthrough( - table_name: str, - target_start: str, - target_end: str, - output_table: Table | None, - capabilities: Capabilities | None = None, - ) -> str: - """Project the target's full row, de-canonicalizing the interval columns. - - NEAREST passes the whole target row through (``SELECT {table_name}.*``) - alongside the synthesized, encoding-invariant ``distance`` column. When the - target's declared encoding is canonical 0-based half-open the row passes - through as a plain ``{table_name}.*`` — the byte-identical identity fast - path. When it is non-canonical the interval columns, canonical inside the - ``__giql_canon_*`` CTE the target was rewritten to, are de-canonicalized - back into that encoding so the passed-through interval matches the target's - own convention. - - The emit strategy is chosen from *capabilities*, following the precedent - of :func:`giql.expanders.disjoin._disjoin_passthrough` (issue #145) — the - same two emit forms, with an added ``capabilities is None`` arm for direct - callers (in production the sole caller always passes ``ctx.capabilities``): - - * ``{table_name}.* REPLACE (...)`` when ``supports_star_replace`` holds (or - no capabilities are supplied) — substitutes start/end in place; - * the portable ``{table_name}.* EXCEPT (start, end), , `` form - otherwise (the generic baseline / DataFusion family). Row-equivalent but - not column-order-equivalent, and not DuckDB-runnable. - - :param table_name: - The relation the row is selected from (the canon CTE name when wrapped, - else the registered table name) — also the column qualifier. - :param target_start: - Physical start column name - :param target_end: - Physical end column name - :param output_table: - The target's declared :class:`~giql.table.Table`, or ``None`` - :param capabilities: - The active target's :class:`~giql.targets.Capabilities`; ``None`` - defaults to the ``* REPLACE`` form. - :return: - The passthrough projection fragment - """ - if output_table is None or ( - output_table.coordinate_system == "0based" - and output_table.interval_type == "half_open" - ): - return f"{table_name}.*" - pt_start = decanonical_start(f'{table_name}."{target_start}"', output_table) - pt_end = decanonical_end(f'{table_name}."{target_end}"', output_table) - if capabilities is None or capabilities.supports_star_replace: - return ( - f"{table_name}.* REPLACE " - f'({pt_start} AS "{target_start}", {pt_end} AS "{target_end}")' - ) - # Portable form for engines without ``* REPLACE`` (generic / DataFusion): - # drop the interval columns from the star and re-project them recomputed. - return ( - f'{table_name}.* EXCEPT ("{target_start}", "{target_end}"), ' - f'{pt_start} AS "{target_start}", {pt_end} AS "{target_end}"' - ) - - @staticmethod - def _generate_distance_case( - chrom_a: str, - start_a: str, - end_a: str, - strand_a: str | None, - chrom_b: str, - start_b: str, - end_b: str, - strand_b: str | None, - stranded: bool = False, - signed: bool = False, - ) -> str: - """Generate SQL CASE expression for distance calculation. - - .. note:: - - KEEP IN SYNC: this method and the AST builder in - ``giql.expanders.distance`` (``expand_distance``) produce the *same* - distance CASE by two routes. DISTANCE itself migrated to the expander - (epic #137, issue #140); this method survives only because NEAREST - still calls it for its ORDER BY / filter math. Once NEAREST migrates, - delete this method and retire the duplication. Until then, any change - to the distance math here must be mirrored in the expander (and vice - versa); the parity test in tests/test_distance_udf.py guards drift. - - Distances follow bedtools ``closest -d`` semantics: overlapping - intervals report ``0``, book-ended (adjacent) intervals where - ``A.end == B.start`` in half-open coordinates report ``1``, and a raw - half-open gap of N bases reports ``N + 1``. The ``+ 1`` is applied to - the absolute gap magnitude before any directional sign, so a downstream - book-ended pair reports ``+1`` and an upstream one ``-1`` in signed mode. - - :param chrom_a: - Chromosome column for interval A - :param start_a: - Start column for interval A - :param end_a: - End column for interval A - :param strand_a: - Strand column for interval A (None if not stranded) - :param chrom_b: - Chromosome column for interval B - :param start_b: - Start column for interval B - :param end_b: - End column for interval B - :param strand_b: - Strand column for interval B (None if not stranded) - :param stranded: - Whether to use strand-aware distance calculation - :param signed: - Whether to return signed distance (negative for upstream, positive for - downstream) - :return: - SQL CASE expression - """ - if not stranded or strand_a is None or strand_b is None: - # Basic distance calculation without strand awareness - if signed: - # Signed distance: negative for upstream (B before A), - # positive for downstream (B after A) - return ( - f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " - f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " - f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " - f"ELSE -({start_a} - {end_b} + 1) END" - ) - # Unsigned (absolute) distance - return ( - f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " - f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " - f"WHEN {end_a} <= {start_b} THEN ({start_b} - {end_a} + 1) " - f"ELSE ({start_a} - {end_b} + 1) END" - ) - - # Stranded distance calculation - # Return NULL if either strand is '.', '?', or NULL - # Calculate distance and multiply by -1 if first interval is on '-' strand - if signed: - # Stranded + signed: apply strand flip AND directional sign - return ( - f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " - f"WHEN {strand_a} IS NULL OR {strand_b} IS NULL THEN NULL " - f"WHEN {strand_a} = '.' OR {strand_a} = '?' THEN NULL " - f"WHEN {strand_b} = '.' OR {strand_b} = '?' THEN NULL " - f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " - f"WHEN {end_a} <= {start_b} THEN " - f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a} + 1) " - f"ELSE ({start_b} - {end_a} + 1) END " - f"ELSE " - f"CASE WHEN {strand_a} = '-' THEN ({start_a} - {end_b} + 1) " - f"ELSE -({start_a} - {end_b} + 1) END END" - ) - # Stranded but not signed: apply strand flip only - return ( - f"CASE WHEN {chrom_a} != {chrom_b} THEN NULL " - f"WHEN {strand_a} IS NULL OR {strand_b} IS NULL THEN NULL " - f"WHEN {strand_a} = '.' OR {strand_a} = '?' THEN NULL " - f"WHEN {strand_b} = '.' OR {strand_b} = '?' THEN NULL " - f"WHEN {start_a} < {end_b} AND {end_a} > {start_b} THEN 0 " - f"WHEN {end_a} <= {start_b} THEN " - f"CASE WHEN {strand_a} = '-' THEN -({start_b} - {end_a} + 1) " - f"ELSE ({start_b} - {end_a} + 1) END " - f"ELSE " - f"CASE WHEN {strand_a} = '-' THEN -({start_a} - {end_b} + 1) " - f"ELSE ({start_a} - {end_b} + 1) END END" - ) - - @staticmethod - def _detect_nearest_mode( - expression: GIQLNearest, parent_expression: exp.Expression | None = None - ) -> str: - """Detect whether NEAREST is in standalone or correlated mode. - - :param expression: - GIQLNearest expression node - :param parent_expression: - Parent AST node (optional, used to detect LATERAL context) - :return: - "standalone" or "correlated" - """ - # Check if reference parameter is explicitly provided - reference = expression.args.get("reference") - - if reference: - # Explicit reference means standalone mode - return "standalone" - - # No explicit reference - check for LATERAL context - # In correlated mode, NEAREST appears in a LATERAL join context - # For now, default to correlated mode if no reference specified - # (validation will catch missing reference errors later) - return "correlated" - - @staticmethod - def _raise_nearest_reference_error( - expression: GIQLNearest, - mode: str, - resolution: OperatorResolution | None, - ) -> None: - """Raise the historical diagnostic for an unresolved NEAREST reference. - - ResolveOperatorRefs (pass 1) defers a reference slot it cannot resolve; - this re-raises the generator's pre-pass error verbatim. The implicit- - outer failures rely on the :class:`~giql.resolver.SlotDeferral` the pass - records (the ancestor walk that distinguished them now lives in the - pass); a literal-range parse failure is reproduced by re-parsing. - - :param expression: - GIQLNearest expression node - :param mode: - "standalone" or "correlated" - :param resolution: - The attached resolution metadata, if any - :raises ValueError: - Always — with the matching historical message - """ - reference = expression.args.get("reference") - - if reference is None: - # Implicit-outer reference: standalone mode (unreachable in practice, - # since an absent reference is classified as correlated) keeps the - # historical message; otherwise consult the recorded deferral. - if mode == "standalone": - raise ValueError( - "NEAREST in standalone mode requires explicit reference parameter" - ) - deferral = ( - resolution.deferral("reference") if resolution is not None else None - ) - if deferral is not None and deferral.reason == "implicit_outer_unregistered": - raise ValueError( - f"Outer table '{deferral.detail}' not found in tables. " - "Please specify reference parameter explicitly." - ) - raise ValueError( - "Could not find outer table in LATERAL join context. " - "Please specify reference parameter explicitly." - ) - - # An explicit reference that deferred is a literal range that failed to - # parse (column references always resolve). Re-parse to surface the - # original parse error in the historical message. - reference_sql = reference.sql(dialect=GIQLDialect) - range_str = reference_sql.strip("'\"") - try: - RangeParser.parse(range_str).to_zero_based_half_open() - except Exception as e: - raise ValueError( - f"Could not parse reference genomic range: {range_str}. Error: {e}" - ) - raise ValueError(f"Could not parse reference genomic range: {range_str}.") - - @staticmethod - def _extract_bool_param(param_expr: exp.Expression | None) -> bool: - """Extract boolean value from a parameter expression. - - Handles :class:`exp.Boolean`, :class:`exp.Literal`, and string - representations. - """ - if not param_expr: - return False - elif isinstance(param_expr, exp.Boolean): - return param_expr.this - else: - return str(param_expr).upper() in ("TRUE", "1", "YES") diff --git a/src/giql/resolver.py b/src/giql/resolver.py index 8431255..be95e1c 100644 --- a/src/giql/resolver.py +++ b/src/giql/resolver.py @@ -491,7 +491,7 @@ def _resolve_operator( def _target_name(target: exp.Expression) -> str: """Extract the target table name from an operator's ``this`` slot. - Mirrors ``BaseGIQLGenerator._resolve_target_table`` exactly. + Mirrors the legacy generator's ``_resolve_target_table`` exactly. """ if isinstance(target, exp.Table): return target.name @@ -969,7 +969,7 @@ def _lookup_aliased_table( def _enclosing_alias_map(node: exp.Expression) -> tuple[dict[str, str], str | None]: """Build the alias->table map of *node*'s enclosing SELECT. - Mirrors ``BaseGIQLGenerator.select_sql`` exactly: the FROM-clause table and + Mirrors the legacy generator's ``select_sql`` exactly: the FROM-clause table and every JOIN-clause table contribute an ``(alias or name) -> name`` entry, and the FROM-clause table name is the ``_current_table`` fallback. Only direct ``exp.Table`` operands participate (a LATERAL/derived join contributes diff --git a/src/giql/targets.py b/src/giql/targets.py index 940804b..83545e2 100644 --- a/src/giql/targets.py +++ b/src/giql/targets.py @@ -35,9 +35,8 @@ class Capabilities: NEAREST LATERAL-vs-window-function strategy (#142): a correlated NEAREST expands to a portable correlated ``LATERAL`` subquery where this holds and to a decorrelated window-function form where it does not. This - capability is the single source of truth — the former - ``BaseGIQLGenerator.SUPPORTS_LATERAL`` generator attribute has been - removed. + capability is the single source of truth — the former generator-level + ``SUPPORTS_LATERAL`` attribute has been removed. supports_star_replace : bool Whether the engine supports ``SELECT * REPLACE (...)``. Drives every coordinate-canonicalization site — the canonicalizer wrapper CTE and the @@ -86,8 +85,8 @@ class GenericTarget(Target): """Portable SQL-92-ish target with no engine-specific features. This is the default target (``dialect=None``). Its capabilities are the - conservative, maximally portable baseline that matches today's - :class:`giql.generators.base.BaseGIQLGenerator` output. + conservative, maximally portable baseline, serialized through sqlglot's + dialect-less default (``sqlglot_dialect = None``). "SQL-92-ish", not strict SQL-92: because ``supports_star_replace=False``, every coordinate-canonicalization site over a **non-canonical** target falls @@ -116,6 +115,15 @@ class DuckDBTarget(Target): Serializes through sqlglot's ``duckdb`` dialect and uses the IEJoin per-partition plan for column-to-column INTERSECTS joins. + + Serializing through the ``duckdb`` dialect makes null ordering **explicit** on + ``ORDER BY`` / window terms, emitting ``NULLS FIRST`` where the generic + (dialect-less) serialization leaves it implicit. This is result-preserving for + every migrated operator because each sorts on a non-null coordinate key + (chromosome / position, or a same-chromosome-filtered distance), so the + explicit ``NULLS FIRST`` never reorders a NULL into or out of the top rows. A + future operator ordering on a nullable key would need to re-verify this before + relying on the DuckDB serialization being a semantic no-op. """ name: str = "duckdb" @@ -179,7 +187,11 @@ def resolve_target(dialect: str | None) -> Target: ---------- dialect : str | None The target dialect name. ``None`` resolves to :class:`GenericTarget`; - ``"duckdb"`` and ``"datafusion"`` resolve to their respective targets. + ``"duckdb"`` and ``"datafusion"`` resolve to their respective built-in + targets. Any other name is resolved against the plugin registry — a + custom :class:`Target` declared through + :meth:`giql.expander.ExpanderRegistry.register_target` (or as a side + effect of :func:`giql.expander.register`) is selectable by its ``name``. Returns ------- @@ -189,14 +201,26 @@ def resolve_target(dialect: str | None) -> Target: Raises ------ ValueError - If *dialect* is not a recognized target name. + If *dialect* is neither a built-in name nor a registered custom target. """ if dialect is None: return GenericTarget() target_cls = _TARGETS_BY_NAME.get(dialect) - if target_cls is None: - raise ValueError( - f"Unknown dialect: {dialect!r}. Supported: 'duckdb', 'datafusion', or None." - ) - return target_cls() + if target_cls is not None: + return target_cls() + + # A non-built-in name may be a custom target registered on the plugin hub. + # Imported lazily so this module stays import-cycle-free: ``giql.expander`` + # imports ``Target`` / ``GenericTarget`` from here at module load. + from giql.expander import REGISTRY + + registered = REGISTRY.target(dialect) + if registered is not None: + return registered + + raise ValueError( + f"Unknown dialect: {dialect!r}. Supported: 'duckdb', 'datafusion', None, " + "or a custom target registered via " + "giql.expander.REGISTRY.register_target()." + ) diff --git a/src/giql/transpile.py b/src/giql/transpile.py index 2607ac9..0dd5d0d 100644 --- a/src/giql/transpile.py +++ b/src/giql/transpile.py @@ -17,7 +17,6 @@ from giql.expander import REGISTRY from giql.expander import ExpandOperators from giql.expressions import Intersects -from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables @@ -46,11 +45,26 @@ def transpile( ) -> str: ... +# Widened overload for arbitrary custom-target dialect names (the registry +# plugin hub). It intentionally subsumes the ``Literal["duckdb"]`` overload +# above, so the static ``dialect="duckdb"`` + ``intersects_bin_size`` guard is not +# enforced for a ``str``-typed dialect; that combination still raises ``ValueError`` +# at runtime (see the body). +@overload +def transpile( + giql: str, + tables: list[str | Table] | None = None, + *, + dialect: str, + intersects_bin_size: int | None = None, +) -> str: ... + + def transpile( giql: str, tables: list[str | Table] | None = None, *, - dialect: Literal["duckdb", "datafusion"] | None = None, + dialect: str | None = None, intersects_bin_size: int | None = None, ) -> str: """Transpile a GIQL query to SQL. @@ -69,10 +83,14 @@ def transpile( Table configurations. Strings use default column mappings (chrom, start, end, strand). :class:`Table` objects provide custom column name mappings. - dialect : Literal["duckdb", "datafusion"] | None + dialect : str | None Optional target engine. Resolves to a :class:`giql.targets.Target` carrying the engine's capability set; ``None`` selects the generic - portable target. When set to ``"duckdb"``, column-to-column + portable target, ``"duckdb"`` / ``"datafusion"`` the built-in targets, + and any other name a custom :class:`giql.targets.Target` registered on + the plugin hub (see :func:`giql.expander.register` / + :meth:`giql.expander.ExpanderRegistry.register_target`). When set to + ``"duckdb"``, column-to-column ``INTERSECTS`` joins (INNER, SEMI, or ANTI) are transpiled into a per-chromosome dynamic-SQL pattern (``SET VARIABLE`` + ``query(getvariable(...))``) that DuckDB plans through its @@ -218,8 +236,6 @@ def transpile( if duckdb_sql is not None: return duckdb_sql - generator = BaseGIQLGenerator(tables=tables_container) - with _reraise_as_value_error("Transformation error"): # Reaching here with an iejoin target means the IEJoin transformer # declined the query (returned None) and fell back to the binned plan, @@ -251,17 +267,20 @@ def transpile( with _reraise_as_value_error("Canonicalization error"): ast = canonicalize_coordinates(ast, target.capabilities) - # Pass 3 of the normalization pipeline (epic #137): replace each GIQL operator - # node that opts in (GIQL_EXPAND) and resolves a registered expander with the - # AST that expander produces for the active target. Operators that are - # unflagged or resolve no expander are left untouched and the generator renders - # them via their legacy ``*_sql`` emitter as before. + # Pass 3 of the normalization pipeline (epic #137): replace every GIQL operator + # node with the AST its registered expander produces for the active target. + # With every operator migrated, this pass fully consumes the GIQL dialect — + # nothing GIQL-specific survives into serialization. expand_pass = ExpandOperators(target, tables_container) with _reraise_as_value_error("Expansion error"): ast = expand_pass.transform(ast) + # Serialize the now-standard AST with the stock sqlglot serializer for the + # active target (epic #137, #146). The target's ``sqlglot_dialect`` selects the + # engine's serialization (``None`` is sqlglot's portable default); there is no + # custom GIQL generator anymore. with _reraise_as_value_error("Transpilation error"): - sql = generator.generate(ast) + sql = ast.sql(dialect=target.sqlglot_dialect) return sql @@ -299,8 +318,9 @@ def _reraise_as_value_error(prefix: str, query: str | None = None) -> Iterator[N """Re-raise non-:class:`ValueError` exceptions as :class:`ValueError` with *prefix*. Lets user-facing :class:`ValueError`\\s from the parser, transformer chain, - and generator propagate verbatim (so the dialect's targeted error messages - survive the boundary) while wrapping unexpected exceptions in a uniform + expander pass, and stock serializer propagate verbatim (so the dialect's + targeted error messages survive the boundary) while wrapping unexpected + exceptions in a uniform :class:`ValueError` prefixed with the stage name. When *query* is supplied, the original input is appended to the message so parse errors retain the offending text. diff --git a/tests/generators/__init__.py b/tests/generators/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/integration/coordinate_space/encodings.py b/tests/integration/coordinate_space/encodings.py index 125bcd2..f2d8009 100644 --- a/tests/integration/coordinate_space/encodings.py +++ b/tests/integration/coordinate_space/encodings.py @@ -5,9 +5,8 @@ integer values that should be stored under each of the four ``(coordinate_system, interval_type)`` combinations supported by GIQL. -The translation is the inverse of -:func:`giql.generators.base.BaseGIQLGenerator._canonical_start` / -``_canonical_end``: re-encoding a canonical interval and feeding the stored +The translation is the inverse of GIQL's coordinate canonicalization (see +:mod:`giql.canonical`): re-encoding a canonical interval and feeding the stored values back through the canonicalization helpers must yield the original canonical values. """ diff --git a/tests/test_canonicalizer.py b/tests/test_canonicalizer.py index d017913..38c8339 100644 --- a/tests/test_canonicalizer.py +++ b/tests/test_canonicalizer.py @@ -26,7 +26,6 @@ from giql.expressions import GIQLDistance from giql.expressions import GIQLNearest from giql.expressions import Intersects -from giql.generators import BaseGIQLGenerator from giql.resolver import META_KEY from giql.resolver import resolve_operator_refs from giql.table import Table @@ -114,7 +113,7 @@ def test_canonical_table_sql_unchanged(self): # now goes through its registered expander, so the pass-bypassed reference # must too, isolating pass 2's contribution to nothing. ast = ExpandOperators(GenericTarget(), tables).transform(ast) - expected = BaseGIQLGenerator(tables=tables).generate(ast) + expected = ast.sql() # Act actual = transpile(query, tables=[Table("variants")]) @@ -146,7 +145,7 @@ def test_non_canonical_table_sql_unchanged(self, monkeypatch): # now expands through its registry entry, so the pass-bypassed reference # must run pass 3 too for the byte-identical comparison to isolate pass 2. ast = ExpandOperators(GenericTarget(), tables).transform(ast) - expected = BaseGIQLGenerator(tables=tables).generate(ast) + expected = ast.sql() # Act actual = transpile(query, tables=[variants]) diff --git a/tests/test_distance_transpilation.py b/tests/test_distance_transpilation.py index 7169d87..4b75e2d 100644 --- a/tests/test_distance_transpilation.py +++ b/tests/test_distance_transpilation.py @@ -10,7 +10,6 @@ from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expander import ExpandOperators -from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables from giql.targets import GenericTarget @@ -32,7 +31,7 @@ def _generate(sql: str, tables: Tables | None = None) -> str: ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) ast = ExpandOperators(GenericTarget(), tables).transform(ast) - return BaseGIQLGenerator(tables=tables).generate(ast) + return ast.sql() class TestDistanceTranspilation: diff --git a/tests/test_distance_udf.py b/tests/test_distance_udf.py index a4a4351..4d6176d 100644 --- a/tests/test_distance_udf.py +++ b/tests/test_distance_udf.py @@ -16,7 +16,7 @@ from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expander import ExpandOperators -from giql.generators import BaseGIQLGenerator +from giql.expanders._distance import generate_distance_case from giql.resolver import resolve_operator_refs from giql.table import Tables from giql.targets import GenericTarget @@ -42,7 +42,7 @@ def _generate(sql: str) -> str: ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) ast = ExpandOperators(GenericTarget(), tables).transform(ast) - return BaseGIQLGenerator().generate(ast) + return ast.sql() def _run(sql: str): @@ -713,13 +713,13 @@ def test_stranded_signed_null_strand_returns_null(self): assert result is None, f"Expected NULL for '.' strand, got {result}" -# --- Drift guard: expand_distance vs the legacy _generate_distance_case ------- +# --- Drift guard: expand_distance vs generate_distance_case ------------------- # -# DISTANCE moved onto the AST-expansion pass (expand_distance), but -# BaseGIQLGenerator._generate_distance_case is retained because NEAREST still -# calls it. The two compute the same distance by different routes; these tests -# pin that they stay semantically equivalent until NEAREST migrates and the -# legacy method can be deleted. +# DISTANCE builds its CASE as AST (expand_distance), but NEAREST assembles its +# SQL string-first and splices in the same CASE via the string builder +# giql.expanders._distance.generate_distance_case. The two compute the same +# distance by different routes; these tests pin that they stay semantically +# equivalent so the string and AST forms cannot silently drift apart. #: Column expressions both routes are evaluated over. ``a``/``b`` are the two #: operand relations supplied by the parity harness's VALUES row. @@ -801,9 +801,9 @@ def _expander_distance_case(stranded: bool, signed: bool) -> str: return ast.find(exp.Select).expressions[0].this.sql() -def _legacy_distance_case(stranded: bool, signed: bool) -> str: - """Return the CASE the legacy _generate_distance_case builds for ``a``/``b``.""" - return BaseGIQLGenerator()._generate_distance_case( +def _string_distance_case(stranded: bool, signed: bool) -> str: + """Return the CASE the string builder generate_distance_case builds for a/b.""" + return generate_distance_case( _CHROM_A, _START_A, _END_A, @@ -844,34 +844,34 @@ def parity_conn(): conn.close() -class TestDistanceExpanderLegacyParity: - """expand_distance and the retained _generate_distance_case agree row-for-row.""" +class TestDistanceExpanderStringParity: + """expand_distance (AST) and generate_distance_case (string) agree row-for-row.""" @pytest.mark.parametrize( "shape_id, stranded, signed", _SHAPES, ids=[s[0] for s in _SHAPES] ) - def test_expander_matches_legacy_distance_case( + def test_expander_matches_string_distance_case( self, parity_conn, shape_id, stranded, signed ): """ GIVEN the four DISTANCE shapes (unsigned/signed x non-stranded/stranded) plus overlap, chrom-mismatch, and strand-invalid input rows - WHEN the same inputs run through expand_distance and the retained - _generate_distance_case + WHEN the same inputs run through expand_distance (AST) and + generate_distance_case (the string builder NEAREST uses) THEN both routes return the identical scalar for every row, pinning the - two distance implementations against drift until NEAREST migrates. + two distance implementations against drift. """ # Arrange expander_case = _expander_distance_case(stranded, signed) - legacy_case = _legacy_distance_case(stranded, signed) + string_case = _string_distance_case(stranded, signed) # Act & assert for row in _PARITY_ROWS: expander_result = _eval_case(parity_conn, expander_case, row) - legacy_result = _eval_case(parity_conn, legacy_case, row) - assert expander_result == legacy_result, ( + string_result = _eval_case(parity_conn, string_case, row) + assert expander_result == string_result, ( f"{shape_id}: expander {expander_result!r} != " - f"legacy {legacy_result!r} for row {row}" + f"string {string_result!r} for row {row}" ) diff --git a/tests/test_expander.py b/tests/test_expander.py index 7844552..5203782 100644 --- a/tests/test_expander.py +++ b/tests/test_expander.py @@ -3,17 +3,18 @@ These tests pin the dispatch infrastructure of epic #137, step 2: * the :class:`~giql.expander.ExpanderRegistry` resolves an expander through the - ``(target, op)`` → ``(generic, op)`` → ``None`` (legacy) fallback chain, and a - later registration overrides an earlier one; + ``(target, op)`` → ``(generic, op)`` → ``None`` fallback chain, and a later + registration overrides an earlier one; * the :func:`~giql.expander.register` decorator writes to the process-wide registry and returns the expander unchanged; -* the :class:`~giql.expander.ExpandOperators` pass dispatches an opted-in - operator to its registered expander — including a *fake* custom target plus a - fake operator, the public extension hook; -* with nothing flagged and an empty registry the pass is a strict no-op and the - emitted SQL is byte-identical. +* the :class:`~giql.expander.ExpandOperators` pass dispatches every operator + node to its registered expander — including a *fake* custom target plus a + fake operator, the public extension hook — and raises when no expander + resolves; +* over an AST carrying no GIQL operators the pass is a strict no-op. """ +from dataclasses import FrozenInstanceError from dataclasses import dataclass import pytest @@ -27,12 +28,12 @@ from giql.expander import ExpandOperators from giql.expander import ExpansionContext from giql.expander import OperatorExpander +from giql.expander import RegistrySnapshot from giql.expander import expand_operators from giql.expander import register from giql.expressions import GIQLDisjoin from giql.expressions import GIQLMerge from giql.expressions import Intersects -from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Table from giql.table import Tables @@ -99,28 +100,6 @@ def _registry_leak_guard(): ) -@pytest.fixture(autouse=True) -def _expand_flag_leak_guard(): - """Assert every operator's GIQL_EXPAND is restored at each test boundary. - - The symmetric partner of the registry leak guard: each operator class ships - a shipped GIQL_EXPAND default (``True`` for a migrated operator, - ``False`` otherwise), and a test that flips one via ``_opted_in`` must restore - it. A leaked flip would silently change the pass for a later test, so this - catches anything that bypasses the exception-safe ``_opted_in`` manager by - comparing against each operator's shipped default rather than a blanket False. - """ - for op in _OPERATOR_CLASSES: - assert op.__dict__.get("GIQL_EXPAND") is _SHIPPED_EXPAND_FLAGS[op], ( - f"{op.__name__}.GIQL_EXPAND leaked into a test from a prior one" - ) - yield - for op in _OPERATOR_CLASSES: - assert op.__dict__.get("GIQL_EXPAND") is _SHIPPED_EXPAND_FLAGS[op], ( - f"a test leaked a GIQL_EXPAND opt-in on {op.__name__}" - ) - - class _CountingExpander: """An expander recording each call so a walk's dispatch count can be pinned.""" @@ -202,7 +181,7 @@ def test_resolve_prefers_target_over_generic(self): # Assert assert resolved is duckdb_fn - def test_resolve_returns_none_for_legacy_fallback(self): + def test_resolve_returns_none_when_no_entry(self): """Test that resolve returns None when no expander is registered. Given: @@ -210,7 +189,8 @@ def test_resolve_returns_none_for_legacy_fallback(self): When: Resolving any (target, op). Then: - It should return None, signalling the legacy *_sql emitter fallback. + It should return None — the pass raises on a miss (there is no + registered expander to dispatch to). """ # Arrange registry = ExpanderRegistry() @@ -372,8 +352,8 @@ def test_snapshot_should_not_observe_later_registrations(self): registry.register(GenericTarget(), Intersects, _record("second")) # Assert - assert (DuckDBTarget(), GIQLDisjoin) in saved - assert (GenericTarget(), Intersects) not in saved + assert (DuckDBTarget(), GIQLDisjoin) in saved.expanders + assert (GenericTarget(), Intersects) not in saved.expanders def test_restore_should_replace_entries_with_snapshot_contents(self): """Test that restore returns the registry to a captured snapshot. @@ -402,6 +382,229 @@ def test_restore_should_replace_entries_with_snapshot_contents(self): assert (GenericTarget(), Intersects) not in registry +@dataclass(frozen=True) +class _PluginTarget(Target): + name: str = "plugin" + sqlglot_dialect: str | None = None + capabilities: Capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=False, + range_join_strategy="binned", + ) + + +class TestTargetRegistration: + """Tests for the registry's target plugin hub (register_target / target).""" + + def test_register_target_makes_custom_target_resolvable(self, clean_registry): + """Test that register_target declares a custom target by name. + + Given: + An empty registry and a custom Target subclass. + When: + Declaring it via register_target. + Then: + REGISTRY.target(name) should return that target. + """ + # Arrange + target = _PluginTarget() + + # Act + clean_registry.register_target(target) + + # Assert + assert clean_registry.target("plugin") is target + + def test_register_auto_declares_target(self, clean_registry): + """Test that registering an expander also declares its target by name. + + Given: + An empty registry and a custom Target subclass. + When: + Registering an expander for (CustomTarget, op). + Then: + REGISTRY.target(name) should return the target — declared as a side + effect of register. + """ + # Act + clean_registry.register(_PluginTarget(), GIQLDisjoin, _record("plugin")) + + # Assert + assert clean_registry.target("plugin") == _PluginTarget() + + def test_register_target_last_write_wins_on_name(self, clean_registry): + """Test that a later target declaration replaces an earlier one by name. + + Given: + Two distinct custom targets sharing the name "plugin". + When: + Declaring the first, then the second, via register_target. + Then: + target("plugin") should return the second (last-write-wins). + """ + + # Arrange + @dataclass(frozen=True) + class _OtherPluginTarget(Target): + name: str = "plugin" + sqlglot_dialect: str | None = "duckdb" + capabilities: Capabilities = Capabilities( + supports_lateral=False, + supports_star_replace=True, + supports_qualify=True, + range_join_strategy="iejoin", + ) + + first = _PluginTarget() + second = _OtherPluginTarget() + + # Act + clean_registry.register_target(first) + clean_registry.register_target(second) + + # Assert + assert clean_registry.target("plugin") is second + + def test_snapshot_restore_round_trips_declared_targets(self, clean_registry): + """Test that snapshot/restore preserves declared targets. + + Given: + A registry with one declared custom target, captured by snapshot and + then cleared. + When: + Restoring the snapshot. + Then: + The custom target should resolve again via target(name). + """ + # Arrange + clean_registry.register_target(_PluginTarget()) + saved = clean_registry.snapshot() + clean_registry.clear() + assert clean_registry.target("plugin") is None + + # Act + clean_registry.restore(saved) + + # Assert + assert clean_registry.target("plugin") == _PluginTarget() + + def test_clear_drops_declared_targets(self, clean_registry): + """Test that clear drops declared targets alongside expanders. + + Given: + A registry with one declared custom target. + When: + Clearing the registry. + Then: + target(name) should return None. + """ + # Arrange + clean_registry.register_target(_PluginTarget()) + + # Act + clean_registry.clear() + + # Assert + assert clean_registry.target("plugin") is None + + +class TestRegistrySnapshot: + """The snapshot value type the leak guard's ``==`` comparison relies on.""" + + def test___eq___equal_for_same_state(self, clean_registry): + """Test that two snapshots of the same registry state compare equal. + + Given: + A registry captured twice with no change in between. + When: + Comparing the two snapshots with ==. + Then: + They should be equal (the leak-guard baseline contract). + """ + # Act & assert + assert clean_registry.snapshot() == clean_registry.snapshot() + + def test___eq___unequal_after_expander_registration(self, clean_registry): + """Test that registering an expander makes a later snapshot unequal. + + Given: + A snapshot of the registry. + When: + Registering an expander, then snapshotting again. + Then: + The two snapshots should compare unequal — so a leak is detectable. + """ + # Arrange + before = clean_registry.snapshot() + + # Act + clean_registry.register(_PluginTarget(), GIQLDisjoin, _record("plugin")) + + # Assert + assert clean_registry.snapshot() != before + + def test___eq___unequal_after_target_only_registration(self, clean_registry): + """Test that a declared-target-only change makes a snapshot unequal. + + Given: + A snapshot of the registry. + When: + Declaring a custom target (no expander), then snapshotting again. + Then: + The two snapshots should compare unequal — the ``targets`` map + contributes to equality, not only ``expanders``. + """ + # Arrange + before = clean_registry.snapshot() + + # Act + clean_registry.register_target(_PluginTarget()) + + # Assert + assert clean_registry.snapshot() != before + + def test___setattr___raises_on_frozen_instance(self, clean_registry): + """Test that a snapshot is an immutable value token. + + Given: + A RegistrySnapshot returned by snapshot(). + When: + Reassigning one of its fields. + Then: + It should raise FrozenInstanceError. + """ + # Arrange + snapshot = clean_registry.snapshot() + + # Act & assert + assert isinstance(snapshot, RegistrySnapshot) + with pytest.raises(FrozenInstanceError): + snapshot.expanders = {} # type: ignore[misc] + + def test_maps_are_independent_copies(self, clean_registry): + """Test that a snapshot is decoupled from later registry mutations. + + Given: + A snapshot taken from the registry. + When: + Registering an expander and declaring a target afterward. + Then: + The snapshot's expanders and targets maps should not observe the + later entries (they are copies, not live views). + """ + # Arrange + snapshot = clean_registry.snapshot() + + # Act + clean_registry.register(_PluginTarget(), GIQLDisjoin, _record("plugin")) + clean_registry.register_target(_PluginTarget()) + + # Assert + assert (_PluginTarget(), GIQLDisjoin) not in snapshot.expanders + assert "plugin" not in snapshot.targets + + class TestExpanderRegistryFallbackGaps: """Edge cases of the registry fallback and op-scoped keying.""" @@ -413,8 +616,8 @@ def test_non_generic_target_with_no_entries_resolves_none(self): When: Resolving (DataFusionTarget(), op). Then: - It resolves to None (legacy emitter fallback), having exhausted the - chain. + It resolves to None, having exhausted the chain (the pass raises on + such a miss). """ # Arrange registry = ExpanderRegistry() @@ -984,11 +1187,11 @@ class TestExpandOperatorsPass: """Tests for the ExpandOperators dispatch pass.""" def test_transform_dispatches_to_registered_expander(self, clean_registry): - """Test that the pass replaces an opted-in node via its expander. + """Test that the pass replaces an operator node via its expander. Given: - A DISJOIN AST whose operator is flagged GIQL_EXPAND and an expander - registered for (GenericTarget, GIQLDisjoin). + A DISJOIN AST and an expander registered for + (GenericTarget, GIQLDisjoin). When: Running the ExpandOperators pass. Then: @@ -1001,8 +1204,7 @@ def test_transform_dispatches_to_registered_expander(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert assert not list(result.find_all(GIQLDisjoin)) @@ -1013,7 +1215,7 @@ def test_transform_dispatches_to_fake_custom_target(self, clean_registry): Given: A fake user target and an expander registered for it via @register, - plus an opted-in DISJOIN operator. + plus a DISJOIN operator. When: Running ExpandOperators with the fake target as active. Then: @@ -1032,8 +1234,7 @@ def _fake_expander(node, ctx): pass_ = ExpandOperators(_FakeTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert assert result.find(exp.Literal).this == "fake-target" @@ -1042,7 +1243,7 @@ def test_transform_passes_resolution_and_target_in_context(self, clean_registry) """Test that the expander receives the pass-1 metadata and active target. Given: - An opted-in DISJOIN with a registered expander that captures its ctx. + A DISJOIN with a registered expander that captures its ctx. When: Running the pass with a DataFusionTarget. Then: @@ -1062,8 +1263,7 @@ def _expander(node, ctx): pass_ = ExpandOperators(DataFusionTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - pass_.transform(ast) + pass_.transform(ast) # Assert ctx = captured["ctx"] @@ -1072,138 +1272,107 @@ def _expander(node, ctx): assert ctx.resolution is not None assert ctx.resolution.operator == "GIQLDisjoin" - def test_transform_skips_unflagged_operator(self, clean_registry): - """Test that an opted-out operator is left untouched even when registered. + def test_transform_expands_operator(self, clean_registry): + """Test that an operator node is replaced by its registered expander. Given: - A migrated operator with an expander registered for it, but its - GIQL_EXPAND flag opted out for the test — so the *same* operator is - registered, queried, and opted out, isolating the per-type gate. + A migrated operator (GIQLDisjoin) with a registered expander. When: Running the pass. Then: - The operator node should remain in the tree (the gate requires the - flag, so opting it out alone holds expansion off). + The operator node should be replaced by the expander's output. """ # Arrange - operator = _A_MIGRATED_OPERATOR + operator = GIQLDisjoin clean_registry.register(GenericTarget(), operator, _record("expanded")) ast, tables = _prepare_operator(operator) pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_out(operator): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert - assert list(result.find_all(operator)) + assert not list(result.find_all(operator)) + assert result.find(exp.Literal).this == "expanded" - def test_transform_expands_flagged_operator(self, clean_registry): - """Test that the same operator expands once flagged (the paired positive). + def test_transform_raises_when_no_expander_resolves(self, clean_registry): + """Test that the pass raises when no expander resolves for an operator. Given: - The same migrated operator and registered expander as the opt-out - control, but with its GIQL_EXPAND flag opted in. + A DISJOIN AST and a cleared registry (no expander resolves). When: Running the pass. Then: - The operator node should be replaced by the expander's output — so the - contrast with the opt-out control pins the per-type gate as - load-bearing, not vacuous. + It should raise ValueError naming the operator type and the target. """ # Arrange - operator = _A_MIGRATED_OPERATOR - clean_registry.register(GenericTarget(), operator, _record("expanded")) - ast, tables = _prepare_operator(operator) - pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - - # Act - with _opted_in(operator): - result = pass_.transform(ast) + REGISTRY.clear() + tables = _tables() + ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + pass_ = ExpandOperators(GenericTarget(), tables) - # Assert - assert not list(result.find_all(operator)) - assert result.find(exp.Literal).this == "expanded" + # Act & assert + with pytest.raises( + ValueError, + match=r"No expander registered for GIQLDisjoin on target 'generic'", + ): + pass_.transform(ast) - def test_transform_skips_flagged_operator_with_no_expander(self, clean_registry): - """Test that a flagged operator with no expander is left untouched. + def test_transform_raise_names_operator_and_custom_target(self, clean_registry): + """Test that the no-expander raise names a custom (non-generic) target. Given: - An opted-in DISJOIN but an empty registry (no expander resolves). + A cleared registry and a DISJOIN AST expanded against a custom target + that has no expander (and no generic fallback). When: - Running the pass. + Running the pass with the custom target active. Then: - The DISJOIN node should remain (gate requires a registered expander). + The ValueError should name both the operator type and the custom + target's name, proving the message interpolation (not a hardcode). """ # Arrange + REGISTRY.clear() tables = _tables() ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) - pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - - # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) + pass_ = ExpandOperators(_PluginTarget(), tables) - # Assert - assert list(result.find_all(GIQLDisjoin)) + # Act & assert + with pytest.raises( + ValueError, + match=r"No expander registered for GIQLDisjoin on target 'plugin'", + ): + pass_.transform(ast) class TestNoOpWhenInert: - """The pass touches nothing while no operator opts in and the registry is empty.""" + """The pass is a strict no-op over an AST that carries no GIQL operators.""" - def test_expand_operators_is_identity_when_registry_empty(self): - """Test that the standalone pass returns the AST unchanged when inert. + def test_expand_operators_is_identity_without_operators(self): + """Test that the standalone pass returns a plain-SQL AST unchanged. Given: - A DISJOIN AST, no operator flagged, and an empty registry. + A plain-SQL AST with no GIQL operator nodes and an empty registry. When: Running expand_operators directly. Then: - The same AST is returned with the DISJOIN node intact. + The same AST is returned unchanged (the pass finds nothing to expand, + so an empty registry never triggers a miss). """ # Arrange tables = _tables() - ast = _prepare("SELECT * FROM DISJOIN(variants)", tables) + ast = _prepare("SELECT * FROM variants", tables) # Act result = expand_operators(ast, GenericTarget(), tables, ExpanderRegistry()) # Assert assert result is ast - assert list(result.find_all(GIQLDisjoin)) - - def test_expand_operators_skips_opted_out_operator_with_default_registry(self): - """Test that the pass skips an opted-out operator even with its expander live. - - Given: - A migrated operator query and the import-populated default REGISTRY (so - its expander resolves), but the operator opted out of GIQL_EXPAND for - the test. - When: - Running the ExpandOperators pass over the default REGISTRY. - Then: - It should leave the operator node in place and return the same tree — - the per-type GIQL_EXPAND gate holds even when an expander is registered - (after #144 no operator ships unmigrated, so the gate is exercised via - an opt-out rather than a shipped False). - """ - # Arrange - operator = _A_MIGRATED_OPERATOR - ast, tables = _prepare_operator(operator) - before_ops = len(list(ast.find_all(operator))) - - # Act — the wired-in pass over the default REGISTRY must be a no-op here. - with _opted_out(operator): - result = expand_operators(ast, GenericTarget(), tables) - - # Assert - assert result is ast - assert len(list(result.find_all(operator))) == before_ops + assert not list(result.find_all(GIQLDisjoin)) # The nine GIQL operator expression classes the ExpandOperators pass inspects. -# Every operator is now migrated (#144): each ships opted in (GIQL_EXPAND=True) -# alongside its registered expander, so none falls through to a legacy emitter. +# Every operator is migrated, so each resolves an expander through the registry; +# this tuple is simply the operator roster the parametrized tests iterate. from giql.expressions import Contains # noqa: E402 from giql.expressions import GIQLCluster # noqa: E402 from giql.expressions import GIQLDistance # noqa: E402 @@ -1223,44 +1392,10 @@ def test_expand_operators_skips_opted_out_operator_with_default_registry(self): GIQLMerge, ) -#: Each operator's shipped GIQL_EXPAND default, captured from its own class dict -#: at import. A migrated operator ships ``True``; the rest ship ``False`` until -#: their migrations land. The flag leak guard restores to these shipped values -#: rather than a blanket ``False``. -# Pin the leak guard's assumption that every operator declares GIQL_EXPAND on its -# own class (so __dict__.get() reads the operator's shipped value rather than an -# inherited one). A future operator that omits its own flag would make the guard -# read ``None`` and silently lose its leak coverage; this one-time check closes -# that hole. -for _op in _OPERATOR_CLASSES: - assert "GIQL_EXPAND" in _op.__dict__, ( - f"{_op.__name__} must declare GIQL_EXPAND on its own class" - ) -_SHIPPED_EXPAND_FLAGS = {op: op.__dict__.get("GIQL_EXPAND") for op in _OPERATOR_CLASSES} - - -#: Operators migrated onto the ExpandOperators pass — they ship GIQL_EXPAND=True. -_MIGRATED_OPERATORS = tuple( - op for op in _OPERATOR_CLASSES if op.__dict__.get("GIQL_EXPAND") is True -) -assert _MIGRATED_OPERATORS, "expected at least one migrated operator" -#: An arbitrary migrated operator the operator-agnostic control tests target. -_A_MIGRATED_OPERATOR = _MIGRATED_OPERATORS[0] -#: Operators not yet migrated — they ship GIQL_EXPAND=False. Empty since #144 -#: migrated the last two (CLUSTER and MERGE): every operator now expands through -#: the pass. Control tests that need an operator to behave as if unmigrated drive a -#: migrated one through ``_opted_out`` rather than relying on a shipped ``False``. -_UNMIGRATED_OPERATORS = tuple( - op for op in _OPERATOR_CLASSES if op not in _MIGRATED_OPERATORS -) -#: Pin the post-#144 invariant: every GIQL operator is migrated onto the pass. -assert not _UNMIGRATED_OPERATORS, "every operator should be migrated after #144" - #: A minimal GIQL query producing one node of each operator class, keyed by the -#: class. Lets the control tests build a node for *any* operator — whichever the -#: branch ships migrated/unmigrated — so they stay operator-agnostic rather than -#: hard-wiring a particular operator. The second element is the table names the +#: class. Lets the operator-agnostic tests build a node for *any* operator rather +#: than hard-wiring a particular class. The second element is the table names the #: query references (registered before pass 1). _OPERATOR_QUERIES: dict[type, tuple[str, tuple[str, ...]]] = { Intersects: ( @@ -1301,11 +1436,10 @@ def test_expand_operators_skips_opted_out_operator_with_default_registry(self): def _prepare_operator(operator: type) -> tuple[exp.Expression, Tables]: """Build a pass-1-resolved AST containing one node of *operator*. - Operator-agnostic: looks the query up in :data:`_OPERATOR_QUERIES` so a - control test can exercise whichever operator a branch ships migrated or - unmigrated, rather than hard-wiring one operator class. Returns the resolved - AST and the Tables container it was resolved against (the same container must - be threaded into the pass). + Operator-agnostic: looks the query up in :data:`_OPERATOR_QUERIES` so a test + can exercise any operator rather than hard-wiring one operator class. Returns + the resolved AST and the Tables container it was resolved against (the same + container must be threaded into the pass). """ query, names = _OPERATOR_QUERIES[operator] tables = _tables(names) @@ -1320,7 +1454,7 @@ def test_transform_replaces_cluster_with_lag_calc_subquery(self): Given: A resolved ``SELECT *, CLUSTER(interval) ...`` AST and the default - REGISTRY (CLUSTER ships GIQL_EXPAND=True with a registered expander). + REGISTRY (CLUSTER has a registered expander). When: Running the ExpandOperators pass. Then: @@ -1357,7 +1491,7 @@ def test_transform_replaces_merge_with_clustered_group_by(self): Given: A resolved ``SELECT MERGE(interval) ...`` AST and the default REGISTRY - (MERGE ships GIQL_EXPAND=True with a registered expander). + (MERGE has a registered expander). When: Running the ExpandOperators pass. Then: @@ -1389,54 +1523,23 @@ def test_transform_replaces_merge_with_clustered_group_by(self): assert EXPAND_ALIAS_PREFIX not in result.sql(dialect=GIQLDialect) -class TestOperatorOptOut: - """Every operator is now migrated, so all ship GIQL_EXPAND=True. - - The complementary ``test_operator_class_ships_expand_disabled`` was dropped - when #144 migrated the last operators: ``_UNMIGRATED_OPERATORS`` is empty, so - there is no class left to assert ships ``False``. - """ - - @pytest.mark.parametrize( - "operator", _MIGRATED_OPERATORS, ids=lambda c: c.__name__ - ) - def test_operator_class_ships_expand_enabled(self, operator): - """Test that each migrated operator class ships GIQL_EXPAND=True. - - Given: - A GIQL operator expression class migrated onto the ExpandOperators - pass. - When: - Reading its GIQL_EXPAND class attribute. - Then: - It should be True (the operator expands through its registered - expander instead of the deleted legacy emitter). - """ - # Arrange & act - flag = operator.GIQL_EXPAND - - # Assert - assert flag is True - - class TestMigratedOperatorsRegistered: - """Every migrated operator resolves a built-in expander in the process REGISTRY.""" + """Every operator resolves a built-in expander in the process REGISTRY.""" @pytest.mark.parametrize( - "operator", _MIGRATED_OPERATORS, ids=lambda c: c.__name__ + "operator", _OPERATOR_CLASSES, ids=lambda c: c.__name__ ) def test_migrated_operator_resolves_in_process_registry(self, operator): - """Test that each migrated operator resolves an expander in REGISTRY. + """Test that each operator resolves an expander in REGISTRY. Given: - A GIQL operator class that ships GIQL_EXPAND=True (migrated onto the - ExpandOperators pass). + A GIQL operator class migrated onto the ExpandOperators pass. When: Resolving it against the import-populated process-wide REGISTRY for the generic target. Then: - A built-in expander should resolve — a migrated operator always has a - registered expander, so the pass never leaves it on a deleted emitter. + A built-in expander should resolve — every operator has a registered + expander, so the pass never leaves it on a deleted emitter. """ # Arrange & act resolved = REGISTRY.resolve(GenericTarget(), operator) @@ -1445,45 +1548,15 @@ def test_migrated_operator_resolves_in_process_registry(self, operator): assert resolved is not None -class TestOptedInRestoresFlag: - """The _opted_in helper restores GIQL_EXPAND even when its body raises.""" - - def test_opted_in_restores_flag_after_exception(self): - """Test that _opted_in restores GIQL_EXPAND when the body raises. - - Given: - An operator driven to GIQL_EXPAND=False via _opted_out (every operator - now ships True after #144, so the restore target is set up explicitly). - When: - Its _opted_in body raises an exception. - Then: - The flag should be restored to False (the manager is exception-safe, - so a raising expansion test cannot leak an opt-in into a later test). - """ - # Arrange — set the restore target to False so the restore is observable. - operator = _A_MIGRATED_OPERATOR - with _opted_out(operator): - assert operator.GIQL_EXPAND is False - - # Act - with pytest.raises(RuntimeError): - with _opted_in(operator): - assert operator.GIQL_EXPAND is True - raise RuntimeError("boom") - - # Assert - assert operator.GIQL_EXPAND is False - - class TestIEJoinRegistryDeferral: """The duckdb IEJoin path defers to a target-specific Intersects expander (#141). Resolves Finding 2: the IEJoin early return used to emit before the - ExpandOperators pass, so a flagged operator on an IEJoin-eligible query was - never expanded. Now a *target-specific* ``(DuckDBTarget, Intersects)`` - registry entry overrides the built-in join strategy entirely (the public - extension hook), while the default duckdb path — with no such override — - still emits the built-in IEJoin SQL. + ExpandOperators pass, so an operator on an IEJoin-eligible query was never + expanded. Now a *target-specific* ``(DuckDBTarget, Intersects)`` registry + entry overrides the built-in join strategy entirely (the public extension + hook), while the default duckdb path — with no such override — still emits the + built-in IEJoin SQL. """ def test_iejoin_query_expands_target_override_expander(self, clean_registry): @@ -1546,8 +1619,7 @@ def test_target_entry_dispatches_for_matching_dialect(self, clean_registry): """Test that a (DuckDBTarget, op) entry dispatches for dialect='duckdb'. Given: - An expander registered for (DuckDBTarget, GIQLDisjoin) and the - operator flagged. + An expander registered for (DuckDBTarget, GIQLDisjoin). When: Transpiling a DISJOIN query with dialect='duckdb'. Then: @@ -1559,10 +1631,9 @@ def test_target_entry_dispatches_for_matching_dialect(self, clean_registry): ) # Act - with _opted_in(GIQLDisjoin): - sql = transpile( - "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect="duckdb" - ) + sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect="duckdb" + ) # Assert assert "DUCK_SENTINEL" in sql @@ -1574,33 +1645,37 @@ def test_target_entry_falls_through_for_other_dialects( """Test that a target-specific entry does not fire for other dialects. Given: - An expander registered only for (DuckDBTarget, GIQLDisjoin), flagged. + An expander registered for (DuckDBTarget, GIQLDisjoin) plus a portable + (GenericTarget, GIQLDisjoin) fallback. When: Transpiling with dialect=None or 'datafusion'. Then: - The sentinel should be absent and the legacy SQL emitted (the - target-specific entry does not resolve for a different target). + The duckdb sentinel is absent and the generic fallback fires instead — + the target-specific entry does not resolve for a different target. """ # Arrange clean_registry.register( DuckDBTarget(), GIQLDisjoin, lambda n, c: exp.column("DUCK_SENTINEL") ) + clean_registry.register( + GenericTarget(), GIQLDisjoin, lambda n, c: exp.column("GENERIC_SENTINEL") + ) # Act - with _opted_in(GIQLDisjoin): - sql = transpile( - "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=dialect - ) + sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=dialect + ) # Assert assert "DUCK_SENTINEL" not in sql + assert "GENERIC_SENTINEL" in sql @pytest.mark.parametrize("dialect", [None, "duckdb", "datafusion"]) def test_generic_entry_covers_all_dialects(self, clean_registry, dialect): """Test that a (GenericTarget, op) entry dispatches via fallback everywhere. Given: - An expander registered only for (GenericTarget, GIQLDisjoin), flagged. + An expander registered only for (GenericTarget, GIQLDisjoin). When: Transpiling with each of the three dialects. Then: @@ -1613,10 +1688,9 @@ def test_generic_entry_covers_all_dialects(self, clean_registry, dialect): ) # Act - with _opted_in(GIQLDisjoin): - sql = transpile( - "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=dialect - ) + sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=dialect + ) # Assert assert "GENERIC_SENTINEL" in sql @@ -1625,7 +1699,7 @@ def test_target_entry_shadows_generic_per_dialect(self, clean_registry): """Test that a target entry shadows the generic entry for its dialect only. Given: - Both a (DuckDBTarget, op) and a (GenericTarget, op) expander, flagged. + Both a (DuckDBTarget, op) and a (GenericTarget, op) expander. When: Transpiling with dialect='duckdb' versus dialect=None. Then: @@ -1641,13 +1715,12 @@ def test_target_entry_shadows_generic_per_dialect(self, clean_registry): ) # Act - with _opted_in(GIQLDisjoin): - duck_sql = transpile( - "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect="duckdb" - ) - generic_sql = transpile( - "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=None - ) + duck_sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect="duckdb" + ) + generic_sql = transpile( + "SELECT * FROM DISJOIN(variants)", tables=["variants"], dialect=None + ) # Assert assert "DUCK_SENTINEL" in duck_sql and "GENERIC_SENTINEL" not in duck_sql @@ -1657,7 +1730,7 @@ def test_context_target_matches_resolved_dialect(self, clean_registry): """Test that ctx.target equals resolve_target(dialect) at dispatch. Given: - A flagged DISJOIN and an expander capturing its context. + A DISJOIN and an expander capturing its context. When: Transpiling with dialect='datafusion'. Then: @@ -1675,12 +1748,11 @@ def _capture(node, ctx): clean_registry.register(GenericTarget(), GIQLDisjoin, _capture) # Act - with _opted_in(GIQLDisjoin): - transpile( - "SELECT * FROM DISJOIN(variants)", - tables=["variants"], - dialect="datafusion", - ) + transpile( + "SELECT * FROM DISJOIN(variants)", + tables=["variants"], + dialect="datafusion", + ) # Assert assert captured["target"] == resolve_target("datafusion") @@ -1689,7 +1761,7 @@ def test_throwing_expander_wraps_in_expansion_error(self, clean_registry): """Test that an unexpected expander error is wrapped as an Expansion error. Given: - A flagged DISJOIN whose expander raises a non-ValueError. + A DISJOIN whose expander raises a non-ValueError. When: Transpiling. Then: @@ -1704,15 +1776,14 @@ def _boom(node, ctx): clean_registry.register(GenericTarget(), GIQLDisjoin, _boom) # Act & assert - with _opted_in(GIQLDisjoin): - with pytest.raises(ValueError, match=r"^Expansion error: "): - transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) + with pytest.raises(ValueError, match=r"^Expansion error: "): + transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) def test_value_error_from_expander_passes_verbatim(self, clean_registry): """Test that a ValueError raised by an expander propagates unwrapped. Given: - A flagged DISJOIN whose expander raises a ValueError with a targeted + A DISJOIN whose expander raises a ValueError with a targeted diagnostic. When: Transpiling. @@ -1728,15 +1799,14 @@ def _raise(node, ctx): clean_registry.register(GenericTarget(), GIQLDisjoin, _raise) # Act & assert - with _opted_in(GIQLDisjoin): - with pytest.raises(ValueError, match=r"^a targeted diagnostic$"): - transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) + with pytest.raises(ValueError, match=r"^a targeted diagnostic$"): + transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) def test_expander_sees_canonicalized_operands(self, clean_registry): """Test that expansion runs after canonicalization (pass ordering). Given: - A flagged DISJOIN with a non-canonical (1-based) target table, which + A DISJOIN with a non-canonical (1-based) target table, which pass 2 wraps in a __giql_canon_ CTE, and an expander capturing its node's resolution at dispatch. When: @@ -1762,8 +1832,7 @@ def _capture(node, ctx): one_based = Table("variants", coordinate_system="1based") # Act - with _opted_in(GIQLDisjoin): - transpile("SELECT * FROM DISJOIN(variants)", tables=[one_based]) + transpile("SELECT * FROM DISJOIN(variants)", tables=[one_based]) # Assert assert captured["resolution"] is not None @@ -1773,7 +1842,7 @@ def test_replacement_reaches_generator(self, clean_registry): """Test that the expander's replacement node is what the generator renders. Given: - A flagged DISJOIN and an expander returning a distinctive sentinel. + A DISJOIN and an expander returning a distinctive sentinel. When: Transpiling. Then: @@ -1786,8 +1855,7 @@ def test_replacement_reaches_generator(self, clean_registry): ) # Act - with _opted_in(GIQLDisjoin): - sql = transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) + sql = transpile("SELECT * FROM DISJOIN(variants)", tables=["variants"]) # Assert assert "REACHED_GENERATOR" in sql @@ -1797,7 +1865,7 @@ def test_unknown_dialect_raises_before_dispatch(self, clean_registry): """Test that an unknown dialect raises before any expander dispatches. Given: - A flagged DISJOIN and a generic expander that would record a call. + A DISJOIN and a generic expander that would record a call. When: Transpiling with an unrecognized dialect. Then: @@ -1809,13 +1877,12 @@ def test_unknown_dialect_raises_before_dispatch(self, clean_registry): clean_registry.register(GenericTarget(), GIQLDisjoin, counting) # Act & assert - with _opted_in(GIQLDisjoin): - with pytest.raises(ValueError, match="Unknown dialect"): - transpile( - "SELECT * FROM DISJOIN(variants)", - tables=["variants"], - dialect="postgres", - ) + with pytest.raises(ValueError, match="Unknown dialect"): + transpile( + "SELECT * FROM DISJOIN(variants)", + tables=["variants"], + dialect="postgres", + ) assert counting.calls == [] @@ -1826,8 +1893,7 @@ def test_walk_visits_every_sibling_operator(self, clean_registry): """Test that the pass dispatches once per sibling operator and clears them. Given: - A query with two sibling DISJOIN operators, flagged, and a counting - expander. + A query with two sibling DISJOIN operators and a counting expander. When: Running the pass. Then: @@ -1844,8 +1910,7 @@ def test_walk_visits_every_sibling_operator(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert assert len(counting.calls) == 2 @@ -1855,8 +1920,7 @@ def test_walk_reaches_nested_operator(self, clean_registry): """Test that the walk reaches an operator nested in a derived table. Given: - A DISJOIN nested inside a derived table, flagged, with a counting - expander. + A DISJOIN nested inside a derived table, with a counting expander. When: Running the pass. Then: @@ -1870,8 +1934,7 @@ def test_walk_reaches_nested_operator(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert assert len(counting.calls) == 1 @@ -1893,7 +1956,7 @@ def test_walk_reaches_operator_in_each_location(self, clean_registry, query): Given: An INTERSECTS predicate placed in a WHERE, JOIN-ON, subquery, or CTE, - flagged, with a counting expander. + with a counting expander. When: Running the pass. Then: @@ -1907,8 +1970,7 @@ def test_walk_reaches_operator_in_each_location(self, clean_registry, query): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(Intersects): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert assert len(counting.calls) == 1 @@ -1918,9 +1980,9 @@ def test_walk_replacement_serializes_through_generator(self, clean_registry): """Test that the pass's replacement serializes cleanly to SQL. Given: - A flagged DISJOIN and an expander returning a column sentinel. + A DISJOIN and an expander returning a column sentinel. When: - Running the pass and serializing the result with BaseGIQLGenerator. + Running the pass and serializing the result with ast.sql(). Then: The sentinel appears in the serialized SQL (replacement integrity). """ @@ -1933,9 +1995,8 @@ def test_walk_replacement_serializes_through_generator(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) - sql = BaseGIQLGenerator(tables=tables).generate(result) + result = pass_.transform(ast) + sql = result.sql() # Assert assert "WALK_SENTINEL" in sql @@ -1944,7 +2005,7 @@ def test_walk_identity_return_leaves_node_in_place(self, clean_registry): """Test that an expander returning the node itself triggers no replacement. Given: - A flagged DISJOIN whose expander returns the node unchanged. + A DISJOIN whose expander returns the node unchanged. When: Running the pass. Then: @@ -1958,8 +2019,7 @@ def test_walk_identity_return_leaves_node_in_place(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert assert list(result.find_all(GIQLDisjoin)) @@ -1968,8 +2028,8 @@ def test_walk_dispatches_mixed_operator_types_per_type(self, clean_registry): """Test that the walk dispatches each operator type to its own expander. Given: - A query containing both a DISJOIN and an INTERSECTS, both flagged, - each with its own counting expander. + A query containing both a DISJOIN and an INTERSECTS, each with its + own counting expander. When: Running the pass. Then: @@ -1989,60 +2049,18 @@ def test_walk_dispatches_mixed_operator_types_per_type(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - with _opted_in(Intersects): - pass_.transform(ast) + pass_.transform(ast) # Assert assert len(disjoin_counter.calls) == 1 assert len(intersects_counter.calls) == 1 - def test_walk_partial_opt_in_replaces_only_flagged_type(self, clean_registry): - """Test that only the flagged operator type is replaced when both registered. - - Given: - A CLUSTER and an INTERSECTS, both with registered expanders, but - CLUSTER held off (opted out of GIQL_EXPAND) while only INTERSECTS is - opted in. - When: - Running the pass. - Then: - The INTERSECTS is replaced while the held-off operator node remains — - the gate is per-type, so opting CLUSTER out alone holds its expansion - off even though its expander is registered. - """ - # Arrange — the held-off subject is a migrated operator driven off via - # _opted_out (after #144 no operator ships GIQL_EXPAND=False). - held_off = GIQLCluster - clean_registry.register( - GenericTarget(), held_off, lambda n, c: exp.column("CL") - ) - clean_registry.register( - GenericTarget(), Intersects, lambda n, c: exp.column("IX") - ) - tables = _tables(("peaks",)) - ast = _prepare( - "SELECT *, CLUSTER(interval) AS cluster_id FROM peaks " - "WHERE EXISTS (SELECT * FROM peaks WHERE interval INTERSECTS 'chr1:1-100')", - tables, - ) - pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) - - # Act — only INTERSECTS is opted in; CLUSTER is held off via opt-out. - with _opted_out(held_off): - with _opted_in(Intersects): - result = pass_.transform(ast) - - # Assert - assert list(result.find_all(held_off)) - assert not list(result.find_all(Intersects)) - def test_walk_shares_alias_sequence_across_sibling_expanders(self, clean_registry): """Test that sibling expanders draw from one alias sequence (no collision). Given: - Two sibling DISJOIN operators, flagged, with an expander that mints an - alias per call and records it. + Two sibling DISJOIN operators with an expander that mints an alias + per call and records it. When: Running the pass. Then: @@ -2066,8 +2084,7 @@ def _mint(node, ctx): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - pass_.transform(ast) + pass_.transform(ast) # Assert assert len(seen) == 2 @@ -2078,11 +2095,10 @@ def test_walk_expands_inner_before_outer_replaces_subtree(self, clean_registry): """Test that a nested operator expands into the live tree before its ancestor. Given: - An outer DISJOIN whose reference subquery contains an inner DISJOIN, - both flagged. The outer's expander returns a fresh node that does NOT - re-attach the inner subtree, and inspects its own subtree so the - recorded order reveals whether the inner had already expanded when the - outer ran. + An outer DISJOIN whose reference subquery contains an inner DISJOIN. + The outer's expander returns a fresh node that does NOT re-attach the + inner subtree, and inspects its own subtree so the recorded order + reveals whether the inner had already expanded when the outer ran. When: Running the pass. Then: @@ -2127,8 +2143,7 @@ def _expander(node, ctx): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act - with _opted_in(GIQLDisjoin): - result = pass_.transform(ast) + result = pass_.transform(ast) # Assert # The inner expands first, into the live tree, so the outer sees it gone. @@ -2139,8 +2154,8 @@ def test_transform_raises_on_node_missing_resolution(self, clean_registry): """Test that the pass raises when an operator lacks resolution metadata. Given: - A flagged DISJOIN whose pass-1 resolution metadata has been stripped - (an internal invariant violation), with a registered expander. + A DISJOIN whose pass-1 resolution metadata has been stripped (an + internal invariant violation), with a registered expander. When: Running the pass. Then: @@ -2159,9 +2174,8 @@ def test_transform_raises_on_node_missing_resolution(self, clean_registry): pass_ = ExpandOperators(GenericTarget(), tables, clean_registry) # Act & assert - with _opted_in(GIQLDisjoin): - with pytest.raises(ResolutionError): - pass_.transform(ast) + with pytest.raises(ResolutionError): + pass_.transform(ast) def test_lazy_import_has_no_cycle_in_fresh_process(self): """Test that importing the expander module standalone raises no cycle. @@ -2170,11 +2184,13 @@ def test_lazy_import_has_no_cycle_in_fresh_process(self): A fresh Python process with nothing pre-imported. When: Importing giql.expander and running its public expand_operators pass - over a parsed operator AST. + over a parsed AST with no GIQL operators (so an empty registry is + never asked to resolve one). Then: - It imports and runs the inert pass without an import cycle, leaving at - least one operator node intact (the empty registry is a no-op), so the - lazy operator import resolves through a public entry point. + It imports and runs the pass without an import cycle — the pass still + imports the operator classes to inspect the tree, so the lazy operator + import resolves through a public entry point — and returns the AST + unchanged. """ # Arrange import os @@ -2189,10 +2205,9 @@ def test_lazy_import_has_no_cycle_in_fresh_process(self): "from giql.targets import GenericTarget; " "from sqlglot import parse_one; " "t = Tables(); t.register('variants', Table('variants')); " - "ast = parse_one('SELECT * FROM DISJOIN(variants)', dialect=GIQLDialect); " + "ast = parse_one('SELECT * FROM variants', dialect=GIQLDialect); " "out = expand_operators(ast, GenericTarget(), t, ExpanderRegistry()); " - "ops = list(out.find_all(GIQLDisjoin)); " - "assert len(ops) >= 1, ops; " + "assert not list(out.find_all(GIQLDisjoin)); " "print('ok')" ) # Strip coverage's subprocess auto-start hooks so the child is a genuinely @@ -2229,41 +2244,3 @@ def _prepare(query: str, tables: Tables) -> exp.Expression: """Parse *query* and run pass 1 (resolution) over the resulting AST.""" ast = parse_one(query, dialect=GIQLDialect) return resolve_operator_refs(ast, tables) - - -class _opted_in: - """Context manager opting an operator class into GIQL_EXPAND for a test.""" - - def __init__(self, operator: type) -> None: - self._operator = operator - self._prior = operator.__dict__.get("GIQL_EXPAND", False) - - def __enter__(self): - self._operator.GIQL_EXPAND = True - return self._operator - - def __exit__(self, *exc): - self._operator.GIQL_EXPAND = self._prior - return False - - -class _opted_out: - """Context manager opting an operator class out of GIQL_EXPAND for a test. - - The complement of :class:`_opted_in`: used by a control test that needs a - *migrated* operator (one shipping GIQL_EXPAND=True) to behave as if - unflagged, so the test can prove the pass gates per-type without the - operator's shipped opt-in interfering. Restores the prior flag on exit. - """ - - def __init__(self, operator: type) -> None: - self._operator = operator - self._prior = operator.__dict__.get("GIQL_EXPAND", False) - - def __enter__(self): - self._operator.GIQL_EXPAND = False - return self._operator - - def __exit__(self, *exc): - self._operator.GIQL_EXPAND = self._prior - return False diff --git a/tests/test_nearest_transpilation.py b/tests/test_nearest_transpilation.py index 9caac74..3520a7e 100644 --- a/tests/test_nearest_transpilation.py +++ b/tests/test_nearest_transpilation.py @@ -12,7 +12,6 @@ from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expander import ExpandOperators -from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables from giql.targets import DataFusionTarget @@ -31,7 +30,7 @@ def _generate_for_target(sql: str, tables: Tables, target) -> str: ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) ast = ExpandOperators(target, tables).transform(ast) - return BaseGIQLGenerator(tables=tables).generate(ast) + return ast.sql() def _generate(sql: str, tables: Tables) -> str: @@ -49,7 +48,7 @@ def _generate(sql: str, tables: Tables) -> str: ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) ast = ExpandOperators(GenericTarget(), tables).transform(ast) - return BaseGIQLGenerator(tables=tables).generate(ast) + return ast.sql() @pytest.fixture @@ -287,6 +286,39 @@ def test_fallback_k_greater_than_one_uses_k_in_topk_predicate( assert "<= 3" in output assert "LATERAL" not in output.upper() + def test_fallback_preserves_signed_and_max_distance( + self, tables_with_peaks_and_genes + ): + """Test the DataFusion fallback keeps signed distance and the max filter. + + Given: + A correlated NEAREST(signed := true, max_distance := 100000) on the + DataFusion target (the decorrelated window path). + When: + Transpiling. + Then: + The window form should retain the signed distance CASE (`ELSE -(`) + and the `<= 100000` max-distance filter — the `signed`/`max_distance` + branches of the shared distance/filter builder on the fallback path. + """ + # Arrange + sql = ( + "SELECT * FROM peaks CROSS JOIN LATERAL NEAREST(" + "genes, reference := peaks.interval, signed := true, " + "max_distance := 100000) AS b" + ) + + # Act + output = _generate_for_target( + sql, tables_with_peaks_and_genes, DataFusionTarget() + ) + + # Assert + assert "ROW_NUMBER(" in output.upper() + assert "LATERAL" not in output.upper() + assert "ELSE -(" in output + assert "<= 100000" in output + class TestNearestUnaliasedCorrelatedFallback: """The fallback synthesizes a LATERAL alias when the user omits one (B3).""" @@ -410,6 +442,6 @@ def test_fallback_detaches_node_and_rewritten_join_survives_pass( # makes the pass's ``node.replace`` a no-op. assert nearest.root() is not result assert not list(result.find_all(GIQLNearest)) - output = BaseGIQLGenerator(tables=tables_with_peaks_and_genes).generate(result) + output = result.sql() assert "LATERAL" not in output.upper() assert "ROW_NUMBER(" in output.upper() diff --git a/tests/test_package_api.py b/tests/test_package_api.py new file mode 100644 index 0000000..6e15e3f --- /dev/null +++ b/tests/test_package_api.py @@ -0,0 +1,95 @@ +"""Tests for the top-level ``giql`` public API surface. + +#146 promoted the extension hook to a supported public API and removed the +custom generator. These tests pin the package facade: the extension-hook +symbols are importable from the ``giql`` root, are the same objects as their +submodule definitions, and the deleted ``giql.generators`` package is gone. +""" + +import importlib + +import pytest + +import giql + +#: The extension-hook symbols #146 added to ``giql.__all__`` (beyond the +#: pre-existing ``DEFAULT_BIN_SIZE`` / ``Table`` / ``transpile``). +_EXTENSION_HOOK_EXPORTS = [ + "register", + "REGISTRY", + "ExpanderRegistry", + "ExpansionContext", + "OperatorExpander", + "Target", + "Capabilities", + "GenericTarget", + "DuckDBTarget", + "DataFusionTarget", +] + +#: Each re-export paired with the submodule it is defined in, so the facade can +#: be checked for object identity (not a shadow copy). +_REEXPORT_ORIGINS = [ + ("register", "giql.expander"), + ("REGISTRY", "giql.expander"), + ("ExpanderRegistry", "giql.expander"), + ("ExpansionContext", "giql.expander"), + ("OperatorExpander", "giql.expander"), + ("Target", "giql.targets"), + ("Capabilities", "giql.targets"), + ("GenericTarget", "giql.targets"), + ("DuckDBTarget", "giql.targets"), + ("DataFusionTarget", "giql.targets"), +] + + +class TestPublicApiSurface: + """The extension hook is a first-class part of the ``giql`` package API.""" + + @pytest.mark.parametrize("name", _EXTENSION_HOOK_EXPORTS) + def test_extension_hook_symbol_is_importable_from_package_root(self, name): + """Test that each extension-hook symbol is exported from ``giql``. + + Given: + An extension-hook name #146 added to ``giql.__all__``. + When: + Accessing it as an attribute of the top-level ``giql`` package and + checking ``giql.__all__``. + Then: + It should resolve to a real attribute and be listed in ``__all__``. + """ + # Act & assert + assert hasattr(giql, name) + assert name in giql.__all__ + + @pytest.mark.parametrize("name, origin_module", _REEXPORT_ORIGINS) + def test_reexport_is_identical_to_submodule_origin(self, name, origin_module): + """Test that a package-root export is the submodule's own object. + + Given: + A name re-exported from ``giql`` and the submodule that defines it. + When: + Comparing ``giql.`` to ``.``. + Then: + They should be the same object (a re-export, not a shadow copy). + """ + # Arrange + submodule = importlib.import_module(origin_module) + + # Act & assert + assert getattr(giql, name) is getattr(submodule, name) + + def test_generators_package_is_removed(self): + """Test that the deleted ``giql.generators`` package is gone. + + Given: + The ``giql.generators`` package that #146 removed. + When: + Attempting to import the package. + Then: + It should raise ModuleNotFoundError, pinning the deletion so the + generator cannot silently reappear. + """ + # Act & assert + with pytest.raises(ModuleNotFoundError): + importlib.import_module("giql.generators") diff --git a/tests/generators/test_base.py b/tests/test_serialization.py similarity index 94% rename from tests/generators/test_base.py rename to tests/test_serialization.py index 73e0d07..d6e6ea0 100644 --- a/tests/generators/test_base.py +++ b/tests/test_serialization.py @@ -1,6 +1,9 @@ -"""Tests for BaseGIQLGenerator. +"""Tests for the stock serialization of expanded GIQL ASTs. -Test specification: specs/test_base.md +Coordinate canonicalization and every operator now expand to standard AST in the +normalization passes; final SQL is produced by the stock sqlglot serializer +(``ast.sql()``) rather than a custom generator (epic #137, #146). These tests +pin that expanded output. """ import pytest @@ -16,7 +19,6 @@ from giql.canonicalizer import canonicalize_coordinates from giql.dialect import GIQLDialect from giql.expander import ExpandOperators -from giql.generators import BaseGIQLGenerator from giql.resolver import resolve_operator_refs from giql.table import Tables from giql.targets import GenericTarget @@ -41,7 +43,7 @@ def _generate_through_passes(sql: str, tables: Tables) -> str: ast = resolve_operator_refs(ast, tables) ast = canonicalize_coordinates(ast) ast = ExpandOperators(GenericTarget(), tables).transform(ast) - return BaseGIQLGenerator(tables=tables).generate(ast) + return ast.sql() @pytest.fixture @@ -101,63 +103,14 @@ def tables_mixed_conventions(): return tables -class TestBaseGIQLGenerator: - """Tests for BaseGIQLGenerator class.""" - - def test_instantiation_defaults(self): - """ - GIVEN no tables provided - WHEN Generator is instantiated with defaults - THEN Generator has empty Tables. - """ - generator = BaseGIQLGenerator() - - assert generator.tables is not None - assert "variants" not in generator.tables - - def test_instantiation_with_tables(self, tables_info): - """ - GIVEN a valid Tables object with table definitions - WHEN Generator is instantiated with tables - THEN Generator stores tables and can resolve column references. - """ - generator = BaseGIQLGenerator(tables=tables_info) - - assert generator.tables is tables_info - assert "variants" in generator.tables - - def test_instantiation_kwargs_forwarding(self): - """ - GIVEN Generator with custom kwargs - WHEN Generator is instantiated with **kwargs - THEN Generator passes kwargs to parent class. - """ - # The parent Generator class accepts various kwargs like 'pretty' - generator = BaseGIQLGenerator(pretty=True) - - # If kwargs forwarding works, generator should have pretty attribute - assert generator.pretty is True - - def test_select_sql_basic(self, tables_info): - """ - GIVEN a SELECT expression with FROM clause containing a table - WHEN select_sql is called - THEN Table context is tracked and alias mapping is built. - """ - sql = "SELECT * FROM variants" - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator(tables=tables_info) - output = generator.generate(ast) - - expected = "SELECT * FROM variants" - assert output == expected +class TestGeneratedSQL: + """Tests for the SQL produced by expanding + stock-serializing GIQL ASTs.""" def test_select_sql_with_alias(self, tables_info): """ - GIVEN a SELECT with aliased table (e.g., FROM table AS t) - WHEN select_sql is called - THEN Alias-to-table mapping includes the alias. + GIVEN an aliased-table INTERSECTS predicate (FROM variants AS v) + WHEN it is expanded and serialized + THEN the predicate expands to alias-qualified range checks. """ sql = "SELECT * FROM variants AS v WHERE v.interval INTERSECTS 'chr1:1000-2000'" @@ -170,25 +123,10 @@ def test_select_sql_with_alias(self, tables_info): ) assert output == expected - def test_select_sql_with_joins(self, tables_with_two_tables): - """ - GIVEN a SELECT with JOINs - WHEN select_sql is called - THEN All joined tables and aliases are tracked. - """ - sql = "SELECT * FROM features_a AS a JOIN features_b AS b ON a.id = b.id" - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) - - expected = "SELECT * FROM features_a AS a JOIN features_b AS b ON a.id = b.id" - assert output == expected - def test_intersects_sql_with_literal(self): """ GIVEN an Intersects expression with literal range 'chr1:1000-2000' - WHEN intersects_sql is called + WHEN it is expanded and serialized THEN SQL with chrom = 'chr1' AND start < 2000 AND end > 1000 is generated. """ sql = "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:1000-2000'" @@ -205,7 +143,7 @@ def test_intersects_sql_column_join(self, tables_with_two_tables): """ GIVEN an Intersects expression with column-to-column (a.interval INTERSECTS b.interval) - WHEN intersects_sql is called + WHEN it is expanded and serialized THEN SQL with column-to-column comparison is generated. """ sql = ( @@ -249,7 +187,7 @@ def test_intersects_sql_validity_property(self, chrom_num, start, length): def test_contains_sql_point_query(self): """ GIVEN a Contains expression with point query 'chr1:1000' - WHEN contains_sql is called + WHEN it is expanded and serialized THEN SQL with start <= 1000 AND end > 1000 is generated. """ sql = "SELECT * FROM variants WHERE interval CONTAINS 'chr1:1500'" @@ -265,7 +203,7 @@ def test_contains_sql_point_query(self): def test_contains_sql_range_query(self): """ GIVEN a Contains expression with range query 'chr1:1000-2000' - WHEN contains_sql is called + WHEN it is expanded and serialized THEN SQL with start <= 1000 AND end >= 2000 is generated. """ sql = "SELECT * FROM variants WHERE interval CONTAINS 'chr1:1500-2000'" @@ -282,7 +220,7 @@ def test_contains_sql_range_query(self): def test_contains_sql_column_join(self, tables_with_two_tables): """ GIVEN a Contains expression with column-to-column join - WHEN contains_sql is called + WHEN it is expanded and serialized THEN SQL with left contains right comparison is generated. """ sql = ( @@ -326,7 +264,7 @@ def test_contains_sql_coordinate_validity_property(self, chrom_num, start, lengt def test_within_sql_with_literal(self): """ GIVEN a Within expression with literal range 'chr1:1000-2000' - WHEN within_sql is called + WHEN it is expanded and serialized THEN SQL with start >= 1000 AND end <= 2000 is generated. """ sql = "SELECT * FROM variants WHERE interval WITHIN 'chr1:1000-5000'" @@ -342,7 +280,7 @@ def test_within_sql_with_literal(self): def test_within_sql_column_join(self, tables_with_two_tables): """ GIVEN a Within expression with column-to-column join - WHEN within_sql is called + WHEN it is expanded and serialized THEN SQL with left within right comparison is generated. """ sql = ( @@ -362,7 +300,7 @@ def test_within_sql_column_join(self, tables_with_two_tables): def test_spatialsetpredicate_sql_any(self): """ GIVEN a SpatialSetPredicate with ANY quantifier and multiple ranges - WHEN spatialsetpredicate_sql is called + WHEN it is expanded and serialized THEN SQL with OR-combined conditions is generated. """ sql = ( @@ -382,7 +320,7 @@ def test_spatialsetpredicate_sql_any(self): def test_spatialsetpredicate_sql_all(self): """ GIVEN a SpatialSetPredicate with ALL quantifier and multiple ranges - WHEN spatialsetpredicate_sql is called + WHEN it is expanded and serialized THEN SQL with AND-combined conditions is generated. """ sql = ( @@ -810,23 +748,6 @@ def test_expand_intersects_should_raise_when_nonnumeric_range_bounds(self): with pytest.raises(ValueError): _generate_through_passes(sql, Tables()) - def test_select_sql_join_without_alias(self, tables_with_two_tables): - """ - GIVEN a SELECT with JOIN where joined table has no alias - WHEN select_sql is called - THEN Table name is used directly in alias mapping. - """ - sql = "SELECT * FROM features_a JOIN features_b ON features_a.id = features_b.id" - ast = parse_one(sql, dialect=GIQLDialect) - - generator = BaseGIQLGenerator(tables=tables_with_two_tables) - output = generator.generate(ast) - - expected = ( - "SELECT * FROM features_a JOIN features_b ON features_a.id = features_b.id" - ) - assert output == expected - def test_expand_nearest_should_use_literal_strand_when_stranded( self, tables_with_peaks_and_genes ): @@ -1027,7 +948,7 @@ def test_expand_nearest_should_raise_when_target_unregistered( def test_intersects_sql_unqualified_column(self): """ GIVEN an unqualified column reference (no table prefix) in spatial operation - WHEN intersects_sql is called + WHEN it is expanded and serialized THEN Default column names are used without table qualifier. """ sql = "SELECT * FROM variants WHERE interval INTERSECTS 'chr1:1000-2000'" diff --git a/tests/test_targets.py b/tests/test_targets.py index 14d10a6..0f0ca25 100644 --- a/tests/test_targets.py +++ b/tests/test_targets.py @@ -2,15 +2,23 @@ import re from dataclasses import FrozenInstanceError +from dataclasses import dataclass import pytest from hypothesis import given from hypothesis import strategies as st +from sqlglot import parse_one +from giql import REGISTRY +from giql import ExpansionContext +from giql import register +from giql import transpile +from giql.expressions import Within from giql.targets import Capabilities from giql.targets import DataFusionTarget from giql.targets import DuckDBTarget from giql.targets import GenericTarget +from giql.targets import Target from giql.targets import resolve_target @@ -326,7 +334,7 @@ def test_resolve_target_with_unsupported_dialect_raises(dialect): pattern = ( re.escape(f"Unknown dialect: {dialect!r}.") + r".*" - + re.escape("'duckdb', 'datafusion', or None.") + + re.escape("'duckdb', 'datafusion', None,") ) with pytest.raises(ValueError, match=pattern): resolve_target(dialect) @@ -347,3 +355,266 @@ def test_resolve_target_with_arbitrary_unsupported_string_raises(dialect): # Act & assert with pytest.raises(ValueError, match="Unknown dialect"): resolve_target(dialect) + + +@dataclass(frozen=True) +class _PostgresTarget(Target): + """A capability-only custom target used to exercise the plugin hub.""" + + name: str = "postgres" + sqlglot_dialect: str | None = "postgres" + capabilities: Capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=True, + range_join_strategy="binned", + ) + + +#: The process-wide REGISTRY state at import (built-in expanders). The custom +#: target tests below mutate the global registry, so — mirroring test_expander.py +#: — an autouse guard asserts the baseline is restored at every test boundary, +#: catching any leaked target/expander (e.g. the "postgres" name this file both +#: registers and, elsewhere, expects to be unresolved) deterministically. +_REGISTRY_BASELINE = REGISTRY.snapshot() + + +@pytest.fixture(autouse=True) +def _registry_leak_guard(): + assert REGISTRY.snapshot() == _REGISTRY_BASELINE, ( + "REGISTRY differed from its baseline entering a test" + ) + yield + assert REGISTRY.snapshot() == _REGISTRY_BASELINE, ( + "a test leaked a registration into REGISTRY" + ) + + +@pytest.fixture +def clean_registry(): + saved = REGISTRY.snapshot() + try: + yield + finally: + REGISTRY.restore(saved) + + +class TestCustomTargetInjection: + """Registering a custom target makes it selectable via ``transpile``.""" + + def test_register_target_makes_custom_target_resolvable(self, clean_registry): + """Test that a declared custom target resolves by name. + + Given: + A custom Target declared through REGISTRY.register_target. + When: + resolve_target is called with its name. + Then: + It should return that exact target instance. + """ + # Arrange + target = _PostgresTarget() + REGISTRY.register_target(target) + + # Act & assert + assert resolve_target("postgres") is target + + def test_transpile_resolves_capability_only_custom_target(self, clean_registry): + """Test end-to-end transpilation against a capability-only custom target. + + Given: + A custom target that overrides no operators, only its capabilities. + When: + A WITHIN query is transpiled with dialect set to its name. + Then: + It should transpile through the generic expanders (its + ``supports_star_replace=False`` selects the portable form) rather + than raising ``Unknown dialect``. + """ + # Arrange + REGISTRY.register_target(_PostgresTarget()) + query = "SELECT * FROM peaks WHERE interval WITHIN 'chr1:1000-5000'" + + # Act + output = transpile(query, tables=["peaks"], dialect="postgres") + + # Assert + assert output == transpile(query, tables=["peaks"]) # generic form + + def test_register_auto_declares_its_target(self, clean_registry): + """Test that registering an expander also declares its target by name. + + Given: + An operator expander registered against a custom target via + @register, with no separate register_target call. + When: + resolve_target is called with the target's name. + Then: + It should resolve the custom target (declared as a side effect). + """ + + # Arrange + @register(_PostgresTarget, Within) + def _noop(node, ctx: ExpansionContext): + return node + + # Act & assert + assert resolve_target("postgres") == _PostgresTarget() + + def test_operator_override_on_custom_target_changes_output(self, clean_registry): + """Test that a per-target expander override reshapes the emitted SQL. + + Given: + A custom target with a WITHIN expander registered that emits a + BETWEEN predicate instead of the generic two-sided comparison. + When: + A WITHIN query is transpiled against that target. + Then: + The emitted SQL should carry the override's BETWEEN form. + """ + + # Arrange + @register(_PostgresTarget, Within) + def _within_between(node, ctx: ExpansionContext): + col = ctx.resolution.column("this") + return parse_one(f"{col.start} BETWEEN 1000 AND 5000") + + query = "SELECT * FROM peaks WHERE interval WITHIN 'chr1:1000-5000'" + + # Act + output = transpile(query, tables=["peaks"], dialect="postgres") + + # Assert + assert "BETWEEN 1000 AND 5000" in output + + def test_register_target_skips_generic(self, clean_registry): + """Test that GenericTarget is never selectable by the name "generic". + + Given: + An attempt to declare GenericTarget on the registry. + When: + resolve_target is called with "generic". + Then: + It should still raise (None is the sole way to select the generic + target), because register_target is a no-op for GenericTarget. + """ + # Arrange + REGISTRY.register_target(GenericTarget()) + + # Act & assert + assert REGISTRY.target("generic") is None + with pytest.raises(ValueError, match="Unknown dialect"): + resolve_target("generic") + + def test_resolve_target_builtin_name_shadows_registry(self, clean_registry): + """Test that a built-in dialect name resolves the built-in, not a shadow. + + Given: + A custom Target registered under a built-in name ("duckdb"). + When: + resolve_target is called with that name. + Then: + It should return the built-in DuckDBTarget — ``_TARGETS_BY_NAME`` is + consulted before the plugin registry, so a custom target cannot + hijack a reserved name. + """ + + # Arrange + @dataclass(frozen=True) + class _ShadowTarget(Target): + name: str = "duckdb" + sqlglot_dialect: str | None = "postgres" + capabilities: Capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=False, + range_join_strategy="binned", + ) + + REGISTRY.register_target(_ShadowTarget()) + + # Act + resolved = resolve_target("duckdb") + + # Assert + assert type(resolved) is DuckDBTarget + + def test_resolve_target_error_names_custom_registration_path(self): + """Test that the unknown-dialect error points at the plugin registry. + + Given: + A dialect name that is neither built-in nor registered. + When: + resolve_target is called with it. + Then: + The ValueError message should mention register_target, guiding the + user toward the custom-target extension path. + """ + # Act & assert + with pytest.raises(ValueError, match="register_target"): + resolve_target("no-such-engine") + + +class TestTargetDrivesSerialization: + """The active target's ``sqlglot_dialect`` selects the stock serializer.""" + + def test_duckdb_dialect_emits_engine_specific_ordering(self): + """Test that the duckdb target serializes through the duckdb dialect. + + Given: + A DISJOIN query whose expansion carries a window ORDER BY. + When: + It is transpiled for the duckdb target versus the generic target. + Then: + The duckdb output should make DuckDB's default null ordering + explicit (``NULLS FIRST``) while the generic (dialect-less) output + should not — confirming ``sqlglot_dialect`` drives serialization. + """ + # Arrange + query = "SELECT * FROM DISJOIN(peaks)" + + # Act + generic_sql = transpile(query, tables=["peaks"]) + duckdb_sql = transpile(query, tables=["peaks"], dialect="duckdb") + + # Assert + assert "NULLS FIRST" in duckdb_sql + assert "NULLS FIRST" not in generic_sql + + def test_custom_target_sqlglot_dialect_drives_serialization(self, clean_registry): + """Test that a registered custom target's sqlglot_dialect reaches ast.sql. + + Given: + A capability-only custom target whose ``sqlglot_dialect="duckdb"``. + When: + A window-carrying operator (DISJOIN) is transpiled under its name + versus the generic (dialect-less) target. + Then: + The custom output should carry the duckdb-dialect ``NULLS FIRST`` + that the generic output lacks — proving a *registered* target's + ``sqlglot_dialect`` is threaded into serialization, not only a + built-in's. + """ + + # Arrange + @dataclass(frozen=True) + class _DuckLikeTarget(Target): + name: str = "ducklike" + sqlglot_dialect: str | None = "duckdb" + capabilities: Capabilities = Capabilities( + supports_lateral=True, + supports_star_replace=False, + supports_qualify=False, + range_join_strategy="binned", + ) + + REGISTRY.register_target(_DuckLikeTarget()) + query = "SELECT * FROM DISJOIN(peaks)" + + # Act + generic_sql = transpile(query, tables=["peaks"]) + custom_sql = transpile(query, tables=["peaks"], dialect="ducklike") + + # Assert + assert "NULLS FIRST" in custom_sql + assert "NULLS FIRST" not in generic_sql diff --git a/tests/test_transpile.py b/tests/test_transpile.py index 754474e..ff7f908 100644 --- a/tests/test_transpile.py +++ b/tests/test_transpile.py @@ -469,6 +469,9 @@ def test_transpile_datafusion_accepted(self): "FROM peaks a, genes b", ["peaks", "genes"], ), + ("SELECT * FROM DISJOIN(peaks)", ["peaks"]), + ("SELECT CLUSTER(interval) FROM peaks", ["peaks"]), + ("SELECT MERGE(interval) FROM peaks", ["peaks"]), ], ids=[ "intersects_literal", @@ -478,6 +481,9 @@ def test_transpile_datafusion_accepted(self): "join", "any", "distance", + "disjoin", + "cluster", + "merge", ], ) def test_transpile_datafusion_matches_generic_output(self, query, tables): @@ -548,6 +554,46 @@ def test_transpile_distance_is_byte_identical_across_targets(self, query, tables # Assert assert generic_sql == duckdb_sql == datafusion_sql + @pytest.mark.parametrize( + "query, tables", + [ + ("SELECT * FROM DISJOIN(peaks)", ["peaks"]), + ( + "SELECT * FROM NEAREST(genes, reference := 'chr1:1000-2000', k := 3)", + ["genes"], + ), + ("SELECT CLUSTER(interval) FROM peaks", ["peaks"]), + ("SELECT MERGE(interval) FROM peaks", ["peaks"]), + ], + ids=["disjoin", "nearest_standalone", "cluster", "merge"], + ) + def test_duckdb_delta_is_only_nulls_ordering(self, query, tables): + """Test that duckdb serialization differs from generic only by NULLS order. + + Given: + A window-carrying operator query. + When: + Transpiling for duckdb and for generic and removing every + ``NULLS FIRST`` / ``NULLS LAST`` token from both outputs. + Then: + The two null-ordering-stripped outputs should be identical — so the + *only* way duckdb serialization diverges from generic is the explicit + null ordering: the duckdb dialect *adds* ``NULLS FIRST`` where generic + leaves it implicit (DISJOIN / NEAREST) and *removes* the redundant + ``NULLS LAST`` the generic form spells out (CLUSTER / MERGE). + """ + + # Arrange + def _strip_nulls(sql): + return sql.replace(" NULLS FIRST", "").replace(" NULLS LAST", "") + + # Act + generic_sql = transpile(query, tables=tables, dialect=None) + duckdb_sql = transpile(query, tables=tables, dialect="duckdb") + + # Assert + assert _strip_nulls(duckdb_sql) == _strip_nulls(generic_sql) + def test_transpile_datafusion_accepts_intersects_bin_size(self): """Test that datafusion honours the binned-join bin size identically. From a474c6f393cd76f55f10aabd77ab74d1a2f62ae4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 2 Jul 2026 16:28:37 -0400 Subject: [PATCH 100/142] feat: Add a statement-finalizer seam and hide NEAREST fallback columns A correlated NEAREST on DataFusion uses a decorrelated ROW_NUMBER fallback whose join must expose reserved rank/key columns for its ON clause; a SELECT * or SELECT b.* over it leaked those internal columns into user output, diverging from the DuckDB LATERAL form's schema. Node-local expanders cannot rewrite the enclosing statement, so this adds a query-level seam: ExpansionContext.add_statement_finalizer registers a StatementFinalizer that expand_operators applies to the statement root after all node-local replacements (it may return a new root). The NEAREST DataFusion fallback registers one that wraps the enclosing SELECT in SELECT * EXCEPT (...) when, and only when, a surfacing star projection would expose the reserved columns, so explicit projections are left untouched. StatementFinalizer is exported from giql for parity with OperatorExpander. Serialization is unchanged for every other query and target; the cross-target identity claim now holds for star projections too. Claude-Session: https://claude.ai/code/session_01ALxmQysPad4W68wuWuft6W --- docs/dialect/distance-operators.rst | 4 +- docs/transpilation/extending.rst | 34 +- src/giql/__init__.py | 2 + src/giql/expander.py | 107 ++++- src/giql/expanders/cluster.py | 19 +- src/giql/expanders/merge.py | 17 +- src/giql/expanders/nearest.py | 143 +++++- src/giql/targets.py | 15 +- .../datafusion/test_cross_target_oracle.py | 156 +++++-- tests/test_expander.py | 232 ++++++++++ tests/test_nearest_transpilation.py | 417 ++++++++++++++++++ tests/test_package_api.py | 10 +- 12 files changed, 1079 insertions(+), 77 deletions(-) diff --git a/docs/dialect/distance-operators.rst b/docs/dialect/distance-operators.rst index ad333d4..b59fc4f 100644 --- a/docs/dialect/distance-operators.rst +++ b/docs/dialect/distance-operators.rst @@ -312,11 +312,11 @@ Find nearby same-strand features within distance constraints: Target support ~~~~~~~~~~~~~~ -A correlated ``NEAREST`` (its reference is an outer-row column) runs on lateral-capable engines — DuckDB and the generic target — via a correlated ``LATERAL`` subquery, and on Apache DataFusion, which has no correlated-``LATERAL`` physical plan, via a decorrelated window-function rewrite. For an **explicitly-projected** query (one that selects named columns, e.g. ``SELECT a.start, b.start, b.distance``) the two forms return identical results: the ``(start, end)`` tiebreaker orders rows tied at the k-th distance the same way on every engine, deterministically whenever ``(start, end)`` distinguishes the tied candidates. A standalone ``NEAREST`` with a literal reference is uncorrelated and uses the same ordered, limited subquery on every target. +A correlated ``NEAREST`` (its reference is an outer-row column) runs on lateral-capable engines — DuckDB and the generic target — via a correlated ``LATERAL`` subquery, and on Apache DataFusion, which has no correlated-``LATERAL`` physical plan, via a decorrelated window-function rewrite. For a single correlated ``NEAREST`` per query the two forms return the same result set, including under ``SELECT *`` / ``SELECT b.*`` (on DataFusion the internal helper columns are projected away — see the note below): the ``(start, end)`` tiebreaker orders rows tied at the k-th distance the same way on every engine, deterministically whenever ``(start, end)`` distinguishes the tied candidates. A trailing top-level ``ORDER BY`` is preserved as a top-level ordering except under the DataFusion star-projection wrapper, which sinks it into the wrapped subquery — there, and only there, rely on the *row set* rather than its order (an explicitly-projected query gets no wrapper on any target, so its ordering is preserved). Not covered — both leak the helper columns on DataFusion, parallel to the residual noted for ``DataFusionTarget``: two correlated ``NEAREST`` fallbacks in one query, and a correlated ``NEAREST`` whose reserved columns are re-surfaced by an enclosing ``SELECT *`` *outside* its own SELECT (e.g. a wrapping ``CLUSTER``). A standalone ``NEAREST`` with a literal reference is uncorrelated and uses the same ordered, limited subquery on every target. .. note:: - **Known limitation —** ``SELECT *`` **/** ``SELECT b.*`` **over a correlated NEAREST on DataFusion.** The decorrelated window-function rewrite needs its reference-key and rank columns (``__giql_x_rk_*``, ``__giql_x_rn``) visible on the rewritten join, so a ``SELECT *`` or ``SELECT b.*`` over a correlated NEAREST exposes those reserved internal columns on DataFusion — a different output schema than the LATERAL form emits on DuckDB. The cross-target identity claim above therefore holds for **explicitly-projected** queries only. Projecting named columns avoids the leak entirely. A query-level wrapper that projects the reserved columns away on the DataFusion path is tracked by `#160 `_ (it depends on the query-level expander seam from #146). + ``SELECT *`` **/** ``SELECT b.*`` **over a correlated NEAREST on DataFusion.** The decorrelated window-function rewrite must expose its reference-key and rank columns (``__giql_x_rk_*``, ``__giql_x_rn``) on the rewritten join for the equi-join to resolve. To keep them out of user output, the DataFusion path wraps the enclosing ``SELECT`` in ``SELECT * EXCEPT () FROM (...)`` (the ``EXCEPT`` list is the explicit reserved column names) when a ``SELECT *`` / ``SELECT b.*`` would surface them, so those projections return the same columns as the DuckDB LATERAL form (#160). Explicitly-projected queries never surface the reserved columns and get no wrapper. Notes ~~~~~ diff --git a/docs/transpilation/extending.rst b/docs/transpilation/extending.rst index 9df4c7c..cf51349 100644 --- a/docs/transpilation/extending.rst +++ b/docs/transpilation/extending.rst @@ -143,11 +143,35 @@ one expression that replaces the operator node in place. It cannot *return* a reshaped enclosing query. An expander may still restructure the query it sits in as a side effect and then return the node unchanged — the built-in CLUSTER and MERGE expanders do exactly this, rewriting their single-table ``SELECT`` in place. -What no expander can express is a rewrite that **adds or reshapes joins** across -relations: the DuckDB IEJoin plan for column-to-column INTERSECTS joins is handled -by a capability-gated pre-pass transformer, not an expander, because it restructures -the surrounding join. A general query-level expander seam for such join rewrites is -planned future work. + +When an expander must rewrite the **enclosing statement** — wrap an enclosing +``SELECT``, or reshape a projection it does not own — it registers a *statement +finalizer* via :meth:`~giql.expander.ExpansionContext.add_statement_finalizer`. +The pass applies every registered finalizer to the statement, in registration +order, after all node-local replacements complete; each receives the current +statement root and returns the (possibly new) root. The built-in NEAREST +DataFusion fallback uses this to wrap its output in +``SELECT * EXCEPT (...)`` and hide the reserved rank/key columns its decorrelated +join must expose: + +.. code-block:: python + + def expand(self, node, ctx): + # ... rewrite the node / enclosing join in place ... + ctx.add_statement_finalizer(lambda root: wrap_or_return(root)) + return node + +A finalizer's returned root is emitted **as-is** — the pass does not re-validate +it — so a finalizer that reshapes a projection must not reference columns or +relations absent from what it rewrites. Wrapping a projection in +``SELECT * EXCEPT (missing_col)``, for instance, builds without error at transpile +time but fails at engine runtime. The built-in fallback guards this by wrapping +only when the projection genuinely surfaces the columns it excepts; a custom +finalizer should apply the same discipline. + +The one query-level rewrite that is *not* an expander is a fold that **adds or +reshapes joins** across relations: the DuckDB IEJoin plan for column-to-column +INTERSECTS joins stays a capability-gated pre-pass transformer by design. Undoing a registration diff --git a/src/giql/__init__.py b/src/giql/__init__.py index a2dce2d..ec26719 100644 --- a/src/giql/__init__.py +++ b/src/giql/__init__.py @@ -8,6 +8,7 @@ from giql.expander import ExpanderRegistry from giql.expander import ExpansionContext from giql.expander import OperatorExpander +from giql.expander import StatementFinalizer from giql.expander import register from giql.table import Table from giql.targets import Capabilities @@ -30,6 +31,7 @@ "ExpanderRegistry", "ExpansionContext", "OperatorExpander", + "StatementFinalizer", "Target", "Capabilities", "GenericTarget", diff --git a/src/giql/expander.py b/src/giql/expander.py index 9809d7b..b01dd70 100644 --- a/src/giql/expander.py +++ b/src/giql/expander.py @@ -71,6 +71,7 @@ "EXPAND_ALIAS_PREFIX", "ExpansionContext", "OperatorExpander", + "StatementFinalizer", "ExpanderRegistry", "RegistrySnapshot", "REGISTRY", @@ -113,9 +114,24 @@ class ExpansionContext: SELECT it just restructured and expand sibling operators it copied into it, honoring a custom-registry pass run. ``None`` for a standalone context built outside the pass. + + A node-local expander that needs to rewrite the *enclosing* statement (rather + than just replace its own node) registers a :data:`StatementFinalizer` via + :meth:`add_statement_finalizer`; the pass applies every finalizer to the + statement after all node-local replacements. This is the query-level seam for + a target whose expansion must reshape the enclosing statement — for example to + project internal helper columns out of a surfacing ``SELECT *``. """ - __slots__ = ("node", "resolution", "target", "tables", "registry", "_alias_seq") + __slots__ = ( + "node", + "resolution", + "target", + "tables", + "registry", + "_alias_seq", + "_finalizers", + ) def __init__( self, @@ -125,6 +141,7 @@ def __init__( tables: Tables, alias_seq: Callable[[], str] | None = None, registry: ExpanderRegistry | None = None, + finalizers: list[StatementFinalizer] | None = None, ) -> None: self.node = node self.resolution = resolution @@ -135,12 +152,38 @@ def __init__( # ``ExpandOperators`` run so aliases minted for sibling operators never # collide; a standalone context falls back to its own sequence. self._alias_seq = alias_seq or name_sequence(EXPAND_ALIAS_PREFIX) + # A single finalizer list is likewise shared across one run's contexts so + # a finalizer registered while expanding one node is applied once, after + # every node-local replacement; a standalone context gets its own (inert + # unless someone drives it manually). + self._finalizers = finalizers if finalizers is not None else [] @property def capabilities(self): """The active target's :class:`~giql.targets.Capabilities`.""" return self.target.capabilities + def add_statement_finalizer(self, finalizer: StatementFinalizer) -> None: + """Register a query-level :data:`StatementFinalizer` for this run. + + The **query-level seam**: an expander is node-local — its return value + replaces only its own node — so a target that must rewrite the *enclosing* + statement (for example to project internal helper columns out of a + surfacing ``SELECT *``) registers a finalizer here instead. + :func:`expand_operators` applies every registered finalizer to the + statement, in registration order, *after* all node-local replacements + complete; each receives the current statement root and returns the + (possibly new) root. + + A finalizer's returned root is emitted verbatim — beyond a type check that + it is an :class:`~sqlglot.expressions.Expression`, it is **not** + semantically re-validated — so it must not reference columns or relations + absent from the projection it rewrites: a wrapper over an absent column + builds without error but fails at engine runtime. Finalizers registered on a standalone context (one built + outside the pass) are collected but never applied. + """ + self._finalizers.append(finalizer) + def alias(self) -> str: """Mint a fresh, query-unique alias with the reserved expander prefix. @@ -164,13 +207,15 @@ class OperatorExpander(Protocol): ``OperatorExpander`` (it has no ``expand`` method); register one by wrapping it (see :func:`register`, which accepts either form). - An expander is **node-local**: ``expand(node, ctx) -> exp.Expression`` sees - one operator node and returns the expression that replaces it in place. It - cannot express a whole-query rewrite such as the INTERSECTS IEJoin fold, - which restructures the surrounding query (joins, CTEs) rather than a single - node. That fold is therefore deferred — it would need a separate - query-level mechanism — and is handled by the pre-pass join transformers, not - by an expander. + An expander's **return value** is node-local: ``expand(node, ctx) -> + exp.Expression`` returns the one expression that replaces the operator node in + place. When a target additionally needs to rewrite the *enclosing* statement — + for example to project internal helper columns away from a surfacing + ``SELECT *`` — the expander registers a query-level :data:`StatementFinalizer` + via :meth:`ExpansionContext.add_statement_finalizer`, applied to the statement + after every node-local replacement. The INTERSECTS IEJoin whole-query fold is a + separate concern still handled by the pre-pass join transformers, not by an + expander. """ def expand(self, node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: ... @@ -180,6 +225,14 @@ def expand(self, node: exp.Expression, ctx: ExpansionContext) -> exp.Expression: #: registry stores either an :class:`OperatorExpander` object or one of these. ExpanderFn = Callable[[exp.Expression, ExpansionContext], exp.Expression] +#: A query-level statement finalizer: ``finalize(root) -> root``. An expander +#: registers one via :meth:`ExpansionContext.add_statement_finalizer` to wrap or +#: rewrite the enclosing statement after every node-local replacement; the pass +#: applies each in registration order and threads the (possibly new) root through. +#: Used, for example, to project internal helper columns out of a surfacing +#: ``SELECT *`` / ``b.*``. +StatementFinalizer = Callable[[exp.Expression], exp.Expression] + def _as_callable(expander: OperatorExpander | ExpanderFn) -> ExpanderFn: """Normalize an expander to a plain ``(node, ctx) -> Expression`` callable.""" @@ -524,7 +577,11 @@ def expand_operators( internal invariant violation (a built-in operator always has at least a ``(generic, op)`` expander) and raises — there is no legacy ``*_sql`` fallback. - The pass mutates and returns *expression* in place. + The pass mutates *expression* in place for the node-local replacements, then + applies any :data:`StatementFinalizer` an expander registered (via + :meth:`ExpansionContext.add_statement_finalizer`) to the statement in + registration order. A finalizer may return a *new* root, so callers must use + the return value rather than assume in-place mutation. Parameters ---------- @@ -541,12 +598,16 @@ def expand_operators( Returns ------- exp.Expression - The same *expression*, with each operator node replaced by its - target-specific expansion. + *expression* with each operator node replaced by its target-specific + expansion, after any registered statement finalizers have run — the same + object when no finalizer replaced the root, otherwise the finalized root. """ reg = registry if registry is not None else REGISTRY operators = _GIQL_OPERATORS alias_seq = name_sequence(EXPAND_ALIAS_PREFIX) + # Shared across every context this run builds: an expander that must rewrite + # the enclosing statement appends a finalizer here, applied after the walk. + finalizers: list[StatementFinalizer] = [] # Collect first, then mutate: replacing nodes mid-walk is unsafe. pending: list[tuple[exp.Expression, ExpanderFn]] = [] @@ -591,7 +652,15 @@ def expand_operators( "valid resolution metadata; pass 1 (resolve_operator_refs) must " "run first and annotate every operator node." ) - ctx = ExpansionContext(node, resolution, target, tables, alias_seq, registry=reg) + ctx = ExpansionContext( + node, + resolution, + target, + tables, + alias_seq, + registry=reg, + finalizers=finalizers, + ) replacement = fn(node, ctx) if not isinstance(replacement, exp.Expression): raise TypeError( @@ -601,6 +670,20 @@ def expand_operators( if replacement is not node: node.replace(replacement) + # Apply any query-level finalizers an expander registered, in registration + # order, once every node-local replacement is in place. A finalizer receives + # the current root and returns the (possibly new) root to thread forward. + for finalize in finalizers: + expression = finalize(expression) + if not isinstance(expression, exp.Expression): + # Mirror the node-local return guard: a finalizer that forgets to + # return the root would otherwise make the pass silently return the + # non-Expression far from the cause. + raise TypeError( + f"statement finalizer {finalize!r} returned " + f"{type(expression).__name__}, not exp.Expression" + ) + return expression diff --git a/src/giql/expanders/cluster.py b/src/giql/expanders/cluster.py index 6285f82..e42b525 100644 --- a/src/giql/expanders/cluster.py +++ b/src/giql/expanders/cluster.py @@ -136,7 +136,20 @@ def expand_cluster(node: GIQLCluster, ctx: ExpansionContext) -> exp.Expression: # re-run the pass over the restructured SELECT to expand any sibling pass-3 # operators (spatial predicates, DISTANCE) carried into it. Safe from # recursion: the CLUSTER node is already replaced by its SUM window. (#144 B1) - expand_operators(select, ctx.target, ctx.tables, ctx.registry) + # + # expand_operators may return a new root (a registered statement finalizer can + # wrap it), and its contract requires callers to use the return value rather + # than assume in-place mutation, so reinstall a new root in place of `select`. + # Today the branch is never taken here — the sole finalizer-registering operator + # (a correlated NEAREST fallback) is already a plain join by this deepest-first + # re-walk, and its wrapper targets an inner SELECT rather than this re-walk root, + # so `result is select` in practice — but honoring the contract keeps the seam + # future-proof. A NEAREST fallback whose reserved columns are re-surfaced by this + # enclosing CLUSTER `SELECT *` remains a documented residual (#172), not a lost + # root. + result = expand_operators(select, ctx.target, ctx.tables, ctx.registry) + if result is not select: + select.replace(result) return node @@ -229,9 +242,7 @@ def find_projected(select: exp.Select, op_type: type[_T]) -> list[_T]: for expression in select.expressions: if isinstance(expression, op_type): found.append(expression) - elif isinstance(expression, exp.Alias) and isinstance( - expression.this, op_type - ): + elif isinstance(expression, exp.Alias) and isinstance(expression.this, op_type): found.append(expression.this) return found diff --git a/src/giql/expanders/merge.py b/src/giql/expanders/merge.py index 4191635..aedea79 100644 --- a/src/giql/expanders/merge.py +++ b/src/giql/expanders/merge.py @@ -97,7 +97,18 @@ def expand_merge(node: GIQLMerge, ctx: ExpansionContext) -> exp.Expression: # subquery; the originals the pass collected are now unreachable, so re-run the # pass over the restructured SELECT to expand any sibling pass-3 operators # carried into it. Safe from recursion: the MERGE is already gone. (#144 B1) - expand_operators(select, ctx.target, ctx.tables, ctx.registry) + # + # expand_operators may return a new root (a registered statement finalizer can + # wrap it), and its contract requires callers to use the return value rather + # than assume in-place mutation, so reinstall a new root in place of `select`. + # Today the branch is never taken here — the sole finalizer-registering operator + # (a correlated NEAREST fallback) is already a plain join by this deepest-first + # re-walk, and MERGE's final projection is explicit, so a nested NEAREST fallback + # never surfaces its reserved columns here anyway — but honoring the contract + # keeps the seam future-proof. + result = expand_operators(select, ctx.target, ctx.tables, ctx.registry) + if result is not select: + select.replace(result) return node @@ -202,9 +213,7 @@ def _transform_for_merge( ) ) select_exprs.append( - exp.alias_( - exp.Max(this=exp.column(end_col, quoted=True)), end_col, quoted=False - ) + exp.alias_(exp.Max(this=exp.column(end_col, quoted=True)), end_col, quoted=False) ) # Process other columns from original SELECT diff --git a/src/giql/expanders/nearest.py b/src/giql/expanders/nearest.py index 8c8e329..7410013 100644 --- a/src/giql/expanders/nearest.py +++ b/src/giql/expanders/nearest.py @@ -33,6 +33,8 @@ from __future__ import annotations +from typing import Callable + from sqlglot import exp from sqlglot import parse_one @@ -41,6 +43,7 @@ from giql.dialect import GIQLDialect from giql.expander import EXPAND_ALIAS_PREFIX from giql.expander import ExpansionContext +from giql.expander import StatementFinalizer from giql.expander import register from giql.expanders._distance import generate_distance_case from giql.expanders._params import coerce_bool_param @@ -176,9 +179,7 @@ def _raise_nearest_reference_error( # An absent reference is a correlated (implicit-outer) placement that the # resolver could not tie to a registered outer table; consult the # recorded deferral for the specific historical message. - deferral = ( - resolution.deferral("reference") if resolution is not None else None - ) + deferral = resolution.deferral("reference") if resolution is not None else None if deferral is not None and deferral.reason == "implicit_outer_unregistered": raise ValueError( f"Outer table '{deferral.detail}' not found in tables. " @@ -343,9 +344,7 @@ def _lateral_form( _abs_distance_expr, where_clauses, passthrough, - ) = _distance_and_filters( - expression, table_name, target_ref, ref, ctx.capabilities - ) + ) = _distance_and_filters(expression, table_name, target_ref, ref, ctx.capabilities) where_sql = " AND ".join(where_clauses) # The wrapping level reads the inner row's *bare* column names (the passthrough # projected ``.*``), so the tiebreaker qualifies them by the wrapper @@ -399,6 +398,108 @@ def _outer_relation(ref: ResolvedInterval) -> tuple[str, str]: return relation, alias +def _projection_surfaces_reserved( + select: exp.Select, b_alias: str, reserved: list[str] +) -> bool: + """Whether *select*'s projection would surface the fallback's reserved columns. + + The decorrelated fallback exposes reserved rank/key columns on the join + relation *b_alias*. They reach the user output only through a projection that + stars over *b*: an unqualified ``*`` (``SELECT *``, which pulls every relation + including *b*) or a ``.*`` (matched case-insensitively unless the + qualifier is explicitly quoted, since engines fold unquoted identifiers). + Returns ``True`` for exactly those two shapes. + + The check must be exact. It must NOT match an explicit projection or a + ``.*`` (e.g. ``a.*``), because those never carry the reserved columns + and wrapping them in ``SELECT * EXCEPT ()`` is not caught at + transpile time — sqlglot builds it happily — but fails at engine runtime. An + unqualified ``*`` that already ``EXCEPT``s every reserved name is treated as + *not* surfacing, so a re-run (or a second finalizer) never double-wraps. + """ + for projection in select.expressions: + if isinstance(projection, exp.Star): + excepted = {col.name for col in (projection.args.get("except_") or [])} + if not set(reserved).issubset(excepted): + return True + elif isinstance(projection, exp.Column) and isinstance( + projection.this, exp.Star + ): + table = projection.args.get("table") + if table is None: + continue + # An unquoted ``B.*`` binds to the same relation as alias ``b`` because + # engines fold unquoted identifiers; match case-insensitively unless the + # qualifier is explicitly quoted (then its case is significant). + quoted = bool(table.args.get("quoted")) + if table.name == b_alias or ( + not quoted and table.name.casefold() == b_alias.casefold() + ): + return True + return False + + +def _wrap_star_except_reserved( + select: exp.Select, reserved: list[str], wrap_alias: str +) -> exp.Select: + """Wrap *select* in ``SELECT * EXCEPT (reserved...) FROM (