Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions src/giql/expanders/_per_chrom.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
"""Shared DuckDB per-chromosome dynamic-SQL scaffolding (#217).

Several DuckDB operator overrides reach the fast ``IE_JOIN`` range-join operator
by partitioning a range join per chromosome, which removes the low-cardinality
``chrom`` equality that would otherwise force a ``HASH_JOIN``. Because GIQL does
not know a table's chromosomes at transpile time, the per-chromosome
``UNION ALL`` is assembled at runtime: a ``string_agg`` over the distinct
chromosomes builds the dynamic SQL into a session variable, which the outer query
reads back through ``query(getvariable(...))``.

This module centralizes the target-agnostic scaffolding — the chromosome
string-literal interpolation, a collision-resistant variable name, the
``SET VARIABLE`` builder, and the dynamic-relation reference. The
operator-specific parts (the per-chromosome SQL template, the chromosome
partition source, the empty-schema fallback, and the outer projection) stay with
each expander. First extracted from
:mod:`giql.expanders.intersects_duckdb`; reused by DISJOIN (#216) and NEAREST.
"""

from __future__ import annotations

from uuid import uuid4

#: A DuckDB expression that renders the ``string_agg`` partition's ``chrom``
#: column as a single-quoted, injection-safe SQL string literal, for
#: interpolation into a per-chromosome ``WHERE chrom = '<c>'`` filter.
CHROM_LITERAL = "'''' || replace(chrom, '''', '''''') || ''''"


def sql_escape(text: str) -> str:
"""Return *text* with single quotes doubled for safe SQL string-literal use."""
return text.replace("'", "''")


def new_variable_name(prefix: str) -> str:
"""Return a collision-resistant DuckDB session-variable name.

The full uuid4 hex (128 bits) keeps the name unique even across many
``transpile()`` calls interleaved in one DuckDB session -- session variables
are global session state, so a token collision would silently rebind a
wrapper query to a different dynamic relation.
"""
return f"{prefix}_{uuid4().hex}"


def set_variable_statement(
var_name: str,
per_chrom_template: str,
chrom_partition: str,
empty_schema: str,
) -> str:
"""Build the ``SET VARIABLE`` statement that assembles the per-chrom dynamic SQL.

*per_chrom_template* is a SQL string-concat expression that ``string_agg``
evaluates once per row of *chrom_partition* (a ``SELECT DISTINCT <chrom>``
source), producing one per-chromosome ``SELECT`` joined by ``UNION ALL``.
When no chromosome passes the partition the aggregate is ``NULL``, so it
falls back to *empty_schema* -- a ``... WHERE FALSE`` query that lets DuckDB
resolve every output column to its real declared type.
"""
return (
f"SET VARIABLE {var_name} = COALESCE((\n"
f" SELECT string_agg(\n"
f" {per_chrom_template},\n"
f" ' UNION ALL '\n"
f" )\n"
f" FROM ({chrom_partition})\n"
f"), '{sql_escape(empty_schema)}')"
)


def dynamic_relation(var_name: str, alias: str) -> str:
"""Return the ``query(getvariable('<var>')) AS <alias>`` relation reference."""
return f"query(getvariable('{var_name}')) AS {alias}"
27 changes: 9 additions & 18 deletions src/giql/expanders/intersects_duckdb.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from dataclasses import dataclass
from typing import ClassVar
from typing import Literal
from uuid import uuid4

from sqlglot import exp

Expand All @@ -38,6 +37,10 @@
from giql.constants import DEFAULT_START_COL
from giql.expander import ExpansionContext
from giql.expander import register
from giql.expanders._per_chrom import CHROM_LITERAL
from giql.expanders._per_chrom import dynamic_relation
from giql.expanders._per_chrom import new_variable_name
from giql.expanders._per_chrom import set_variable_statement
from giql.expanders.intersects import SPATIAL_PREDICATE_META
from giql.expanders.intersects import _expand_spatial_op
from giql.expressions import Contains
Expand Down Expand Up @@ -1209,13 +1212,7 @@ def _build_sql(
outer_where_residuals,
)

# Per-call random token (full uuid4 hex = 128 bits) so the SET
# VARIABLE name is collision-resistant even across many transpile()
# calls interleaved in one DuckDB session. DuckDB session variables
# are global session state, so token collision would silently
# rebind the wrapper query to a different intersection.
token = uuid4().hex
var_name = f"__giql_iejoin_{token}"
var_name = new_variable_name("__giql_iejoin")

# Canonicalize endpoints to 0-based half-open and use strict
# operators. Mixing inclusive operators with raw closed-closed
Expand Down Expand Up @@ -1250,7 +1247,7 @@ def _build_sql(
# doubled because the result is itself an SQL string literal that
# contains SQL. The chromosome literal is interpolated twice — once
# into the left partition filter, once into the right.
chrom_literal = "'''' || replace(chrom, '''', '''''') || ''''"
chrom_literal = CHROM_LITERAL
esc = self._sql_escape
inner_select_list = ", ".join(inner_projections)

Expand Down Expand Up @@ -1332,14 +1329,8 @@ def _build_sql(
f"FROM {r_table_ident}"
)

set_var_stmt = (
f"SET VARIABLE {var_name} = COALESCE((\n"
f" SELECT string_agg(\n"
f" {per_chrom_sql_expr},\n"
f" ' UNION ALL '\n"
f" )\n"
f" FROM ({chrom_partition_subquery})\n"
f"), '{esc(empty_schema)}')"
set_var_stmt = set_variable_statement(
var_name, per_chrom_sql_expr, chrom_partition_subquery, empty_schema
)

# Constant wrapper alias — uniqueness is provided by the
Expand All @@ -1350,7 +1341,7 @@ def _build_sql(
distinct_kw = "DISTINCT " if query.args.get("distinct") else ""
outer_select_parts = [
f"SELECT {distinct_kw}{outer_projection} "
f"FROM query(getvariable('{var_name}')) AS {wrapper_alias}"
f"FROM {dynamic_relation(var_name, wrapper_alias)}"
]

# WHERE residuals for the left-only (SEMI / ANTI) shapes are applied
Expand Down
Loading