From c7ca4dbd485552b3d4c7ae041c78276de53ee253 Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 9 Jul 2026 09:48:24 -0400 Subject: [PATCH] refactor: Extract the per-chromosome dynamic-SQL scaffolding to a shared helper The DuckDB per-chromosome dynamic-SQL machinery -- the chromosome string-literal interpolation, the collision-resistant session-variable name, the SET VARIABLE ... = COALESCE((string_agg per chromosome), empty) builder, and the query(getvariable(...)) relation reference -- moves from IntersectsDuckDBIEJoinTransformer into a new giql.expanders._per_chrom module. The operator-specific parts (the per-chromosome SQL template, the partition source, the empty-schema fallback, and the outer projection) stay in the expander. Emitted SQL is byte-identical (no behavior change), readying the scaffolding for reuse by DISJOIN (#216) and NEAREST. Claude-Session: https://claude.ai/code/session_01TERWBHov76DQM3nQeT8yyy --- src/giql/expanders/_per_chrom.py | 74 +++++++++++++++++++++++++ src/giql/expanders/intersects_duckdb.py | 27 +++------ 2 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 src/giql/expanders/_per_chrom.py diff --git a/src/giql/expanders/_per_chrom.py b/src/giql/expanders/_per_chrom.py new file mode 100644 index 0000000..be5f96b --- /dev/null +++ b/src/giql/expanders/_per_chrom.py @@ -0,0 +1,74 @@ +"""Shared DuckDB per-chromosome dynamic-SQL scaffolding (#217). + +Several DuckDB operator overrides reach the fast ``IE_JOIN`` range-join operator +by partitioning a range join per chromosome, which removes the low-cardinality +``chrom`` equality that would otherwise force a ``HASH_JOIN``. Because GIQL does +not know a table's chromosomes at transpile time, the per-chromosome +``UNION ALL`` is assembled at runtime: a ``string_agg`` over the distinct +chromosomes builds the dynamic SQL into a session variable, which the outer query +reads back through ``query(getvariable(...))``. + +This module centralizes the target-agnostic scaffolding — the chromosome +string-literal interpolation, a collision-resistant variable name, the +``SET VARIABLE`` builder, and the dynamic-relation reference. The +operator-specific parts (the per-chromosome SQL template, the chromosome +partition source, the empty-schema fallback, and the outer projection) stay with +each expander. First extracted from +:mod:`giql.expanders.intersects_duckdb`; reused by DISJOIN (#216) and NEAREST. +""" + +from __future__ import annotations + +from uuid import uuid4 + +#: A DuckDB expression that renders the ``string_agg`` partition's ``chrom`` +#: column as a single-quoted, injection-safe SQL string literal, for +#: interpolation into a per-chromosome ``WHERE chrom = ''`` filter. +CHROM_LITERAL = "'''' || replace(chrom, '''', '''''') || ''''" + + +def sql_escape(text: str) -> str: + """Return *text* with single quotes doubled for safe SQL string-literal use.""" + return text.replace("'", "''") + + +def new_variable_name(prefix: str) -> str: + """Return a collision-resistant DuckDB session-variable name. + + The full uuid4 hex (128 bits) keeps the name unique even across many + ``transpile()`` calls interleaved in one DuckDB session -- session variables + are global session state, so a token collision would silently rebind a + wrapper query to a different dynamic relation. + """ + return f"{prefix}_{uuid4().hex}" + + +def set_variable_statement( + var_name: str, + per_chrom_template: str, + chrom_partition: str, + empty_schema: str, +) -> str: + """Build the ``SET VARIABLE`` statement that assembles the per-chrom dynamic SQL. + + *per_chrom_template* is a SQL string-concat expression that ``string_agg`` + evaluates once per row of *chrom_partition* (a ``SELECT DISTINCT `` + source), producing one per-chromosome ``SELECT`` joined by ``UNION ALL``. + When no chromosome passes the partition the aggregate is ``NULL``, so it + falls back to *empty_schema* -- a ``... WHERE FALSE`` query that lets DuckDB + resolve every output column to its real declared type. + """ + return ( + f"SET VARIABLE {var_name} = COALESCE((\n" + f" SELECT string_agg(\n" + f" {per_chrom_template},\n" + f" ' UNION ALL '\n" + f" )\n" + f" FROM ({chrom_partition})\n" + f"), '{sql_escape(empty_schema)}')" + ) + + +def dynamic_relation(var_name: str, alias: str) -> str: + """Return the ``query(getvariable('')) AS `` relation reference.""" + return f"query(getvariable('{var_name}')) AS {alias}" diff --git a/src/giql/expanders/intersects_duckdb.py b/src/giql/expanders/intersects_duckdb.py index 126d87d..f5bbf09 100644 --- a/src/giql/expanders/intersects_duckdb.py +++ b/src/giql/expanders/intersects_duckdb.py @@ -27,7 +27,6 @@ from dataclasses import dataclass from typing import ClassVar from typing import Literal -from uuid import uuid4 from sqlglot import exp @@ -38,6 +37,10 @@ from giql.constants import DEFAULT_START_COL from giql.expander import ExpansionContext from giql.expander import register +from giql.expanders._per_chrom import CHROM_LITERAL +from giql.expanders._per_chrom import dynamic_relation +from giql.expanders._per_chrom import new_variable_name +from giql.expanders._per_chrom import set_variable_statement from giql.expanders.intersects import SPATIAL_PREDICATE_META from giql.expanders.intersects import _expand_spatial_op from giql.expressions import Contains @@ -1209,13 +1212,7 @@ def _build_sql( outer_where_residuals, ) - # Per-call random token (full uuid4 hex = 128 bits) so the SET - # VARIABLE name is collision-resistant even across many transpile() - # calls interleaved in one DuckDB session. DuckDB session variables - # are global session state, so token collision would silently - # rebind the wrapper query to a different intersection. - token = uuid4().hex - var_name = f"__giql_iejoin_{token}" + var_name = new_variable_name("__giql_iejoin") # Canonicalize endpoints to 0-based half-open and use strict # operators. Mixing inclusive operators with raw closed-closed @@ -1250,7 +1247,7 @@ def _build_sql( # doubled because the result is itself an SQL string literal that # contains SQL. The chromosome literal is interpolated twice — once # into the left partition filter, once into the right. - chrom_literal = "'''' || replace(chrom, '''', '''''') || ''''" + chrom_literal = CHROM_LITERAL esc = self._sql_escape inner_select_list = ", ".join(inner_projections) @@ -1332,14 +1329,8 @@ def _build_sql( f"FROM {r_table_ident}" ) - set_var_stmt = ( - f"SET VARIABLE {var_name} = COALESCE((\n" - f" SELECT string_agg(\n" - f" {per_chrom_sql_expr},\n" - f" ' UNION ALL '\n" - f" )\n" - f" FROM ({chrom_partition_subquery})\n" - f"), '{esc(empty_schema)}')" + set_var_stmt = set_variable_statement( + var_name, per_chrom_sql_expr, chrom_partition_subquery, empty_schema ) # Constant wrapper alias — uniqueness is provided by the @@ -1350,7 +1341,7 @@ def _build_sql( distinct_kw = "DISTINCT " if query.args.get("distinct") else "" outer_select_parts = [ f"SELECT {distinct_kw}{outer_projection} " - f"FROM query(getvariable('{var_name}')) AS {wrapper_alias}" + f"FROM {dynamic_relation(var_name, wrapper_alias)}" ] # WHERE residuals for the left-only (SEMI / ANTI) shapes are applied