Skip to content

Commit 803f23d

Browse files
jackylee-chclaude
andcommitted
Reject out-of-range literals when binding IN / NOT IN
`SetPredicate.bind` kept the result of `Literal.to(field_type)` for every literal. For a value outside the field's range that result is the `AboveMax`/`BelowMin` sentinel, whose value is the type's max/min, so the bound set held a literal the user never wrote. On an `int` column, `id in (1, 2**40)` matched rows where `id` equals 2147483647, and the `not in` form dropped them. `LiteralPredicate.bind` already folds these to `AlwaysTrue`/`AlwaysFalse`, and Java's `bindInOperation` filters them out of the set; do the same here. An empty set after filtering is already folded by `BoundIn`/`BoundNotIn`. Co-Authored-By: Claude Code <noreply@anthropic.com>
1 parent 7a7a99b commit 803f23d

2 files changed

Lines changed: 37 additions & 2 deletions

File tree

‎pyiceberg/expressions/__init__.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -697,8 +697,13 @@ def __init__(
697697

698698
def bind(self, schema: Schema, case_sensitive: bool = True) -> BoundSetPredicate:
699699
bound_term = self.term.bind(schema, case_sensitive)
700-
literal_set = self.literals
701-
return self.as_bound(bound_term, {lit.to(bound_term.ref().field.field_type) for lit in literal_set}) # type: ignore
700+
field_type = bound_term.ref().field.field_type
701+
# Literals outside the field's range can never match, so drop them rather
702+
# than keep the clamped AboveMax/BelowMin sentinel in the bound set
703+
bound_literals = {lit.to(field_type) for lit in self.literals}
704+
return self.as_bound( # type: ignore
705+
bound_term, {lit for lit in bound_literals if not isinstance(lit, (AboveMax, BelowMin))}
706+
)
702707

703708
def __str__(self) -> str:
704709
"""Return the string representation of the SetPredicate class."""

‎tests/expressions/test_evaluator.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424

2525
from pyiceberg.conversions import to_bytes
2626
from pyiceberg.expressions import (
27+
AlwaysFalse,
28+
AlwaysTrue,
2729
And,
2830
BooleanExpression,
2931
EqualTo,
@@ -51,6 +53,7 @@
5153
ROWS_MUST_MATCH,
5254
_InclusiveMetricsEvaluator,
5355
_StrictMetricsEvaluator,
56+
expression_evaluator,
5457
)
5558
from pyiceberg.manifest import DataFile, FileFormat
5659
from pyiceberg.schema import Schema
@@ -1907,3 +1910,30 @@ def test_strict_metrics_eval_bounds_after_promotion(
19071910

19081911
evaluator = _StrictMetricsEvaluator(schema, op("col", lit))
19091912
assert evaluator.eval(data_file) == expected
1913+
1914+
1915+
def test_above_int_bounds_in() -> None:
1916+
schema = Schema(NestedField(1, "id", IntegerType(), required=False))
1917+
above_max = IntegerType.max + 1
1918+
1919+
assert In("id", [1, above_max]).bind(schema) == EqualTo("id", 1).bind(schema)
1920+
assert NotIn("id", [1, above_max]).bind(schema) == NotEqualTo("id", 1).bind(schema)
1921+
assert In("id", [above_max]).bind(schema) == AlwaysFalse()
1922+
assert NotIn("id", [above_max]).bind(schema) == AlwaysTrue()
1923+
1924+
# The clamped literal used to match the field's maximum
1925+
assert expression_evaluator(schema, In("id", [1, above_max]), True)(Record(IntegerType.max)) is False
1926+
assert expression_evaluator(schema, NotIn("id", [1, above_max]), True)(Record(IntegerType.max)) is True
1927+
1928+
1929+
def test_below_int_bounds_in() -> None:
1930+
schema = Schema(NestedField(1, "id", IntegerType(), required=False))
1931+
below_min = IntegerType.min - 1
1932+
1933+
assert In("id", [1, below_min]).bind(schema) == EqualTo("id", 1).bind(schema)
1934+
assert NotIn("id", [1, below_min]).bind(schema) == NotEqualTo("id", 1).bind(schema)
1935+
assert In("id", [below_min]).bind(schema) == AlwaysFalse()
1936+
assert NotIn("id", [below_min]).bind(schema) == AlwaysTrue()
1937+
1938+
assert expression_evaluator(schema, In("id", [1, below_min]), True)(Record(IntegerType.min)) is False
1939+
assert expression_evaluator(schema, NotIn("id", [1, below_min]), True)(Record(IntegerType.min)) is True

0 commit comments

Comments
 (0)